1/**************************************************************************** 2 * Driver for Solarflare network controllers and boards 3 * Copyright 2005-2006 Fen Systems Ltd. 4 * Copyright 2006-2013 Solarflare Communications Inc. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 as published 8 * by the Free Software Foundation, incorporated herein by reference. 9 */ 10 11#include <linux/bitops.h> 12#include <linux/delay.h> 13#include <linux/interrupt.h> 14#include <linux/pci.h> 15#include <linux/module.h> 16#include <linux/seq_file.h> 17#include <linux/crc32.h> 18#include "net_driver.h" 19#include "bitfield.h" 20#include "efx.h" 21#include "nic.h" 22#include "farch_regs.h" 23#include "io.h" 24#include "workarounds.h" 25 26/* Falcon-architecture (SFC4000 and SFC9000-family) support */ 27 28/************************************************************************** 29 * 30 * Configurable values 31 * 32 ************************************************************************** 33 */ 34 35/* This is set to 16 for a good reason. In summary, if larger than 36 * 16, the descriptor cache holds more than a default socket 37 * buffer's worth of packets (for UDP we can only have at most one 38 * socket buffer's worth outstanding). This combined with the fact 39 * that we only get 1 TX event per descriptor cache means the NIC 40 * goes idle. 41 */ 42#define TX_DC_ENTRIES 16 43#define TX_DC_ENTRIES_ORDER 1 44 45#define RX_DC_ENTRIES 64 46#define RX_DC_ENTRIES_ORDER 3 47 48/* If EFX_MAX_INT_ERRORS internal errors occur within 49 * EFX_INT_ERROR_EXPIRE seconds, we consider the NIC broken and 50 * disable it. 51 */ 52#define EFX_INT_ERROR_EXPIRE 3600 53#define EFX_MAX_INT_ERRORS 5 54 55/* Depth of RX flush request fifo */ 56#define EFX_RX_FLUSH_COUNT 4 57 58/* Driver generated events */ 59#define _EFX_CHANNEL_MAGIC_TEST 0x000101 60#define _EFX_CHANNEL_MAGIC_FILL 0x000102 61#define _EFX_CHANNEL_MAGIC_RX_DRAIN 0x000103 62#define _EFX_CHANNEL_MAGIC_TX_DRAIN 0x000104 63 64#define _EFX_CHANNEL_MAGIC(_code, _data) ((_code) << 8 | (_data)) 65#define _EFX_CHANNEL_MAGIC_CODE(_magic) ((_magic) >> 8) 66 67#define EFX_CHANNEL_MAGIC_TEST(_channel) \ 68 _EFX_CHANNEL_MAGIC(_EFX_CHANNEL_MAGIC_TEST, (_channel)->channel) 69#define EFX_CHANNEL_MAGIC_FILL(_rx_queue) \ 70 _EFX_CHANNEL_MAGIC(_EFX_CHANNEL_MAGIC_FILL, \ 71 efx_rx_queue_index(_rx_queue)) 72#define EFX_CHANNEL_MAGIC_RX_DRAIN(_rx_queue) \ 73 _EFX_CHANNEL_MAGIC(_EFX_CHANNEL_MAGIC_RX_DRAIN, \ 74 efx_rx_queue_index(_rx_queue)) 75#define EFX_CHANNEL_MAGIC_TX_DRAIN(_tx_queue) \ 76 _EFX_CHANNEL_MAGIC(_EFX_CHANNEL_MAGIC_TX_DRAIN, \ 77 (_tx_queue)->queue) 78 79static void efx_farch_magic_event(struct efx_channel *channel, u32 magic); 80 81/************************************************************************** 82 * 83 * Hardware access 84 * 85 **************************************************************************/ 86 87static inline void efx_write_buf_tbl(struct efx_nic *efx, efx_qword_t *value, 88 unsigned int index) 89{ 90 efx_sram_writeq(efx, efx->membase + efx->type->buf_tbl_base, 91 value, index); 92} 93 94static bool efx_masked_compare_oword(const efx_oword_t *a, const efx_oword_t *b, 95 const efx_oword_t *mask) 96{ 97 return ((a->u64[0] ^ b->u64[0]) & mask->u64[0]) || 98 ((a->u64[1] ^ b->u64[1]) & mask->u64[1]); 99} 100 101int efx_farch_test_registers(struct efx_nic *efx, 102 const struct efx_farch_register_test *regs, 103 size_t n_regs) 104{ 105 unsigned address = 0, i, j; 106 efx_oword_t mask, imask, original, reg, buf; 107 108 for (i = 0; i < n_regs; ++i) { 109 address = regs[i].address; 110 mask = imask = regs[i].mask; 111 EFX_INVERT_OWORD(imask); 112 113 efx_reado(efx, &original, address); 114 115 /* bit sweep on and off */ 116 for (j = 0; j < 128; j++) { 117 if (!EFX_EXTRACT_OWORD32(mask, j, j)) 118 continue; 119 120 /* Test this testable bit can be set in isolation */ 121 EFX_AND_OWORD(reg, original, mask); 122 EFX_SET_OWORD32(reg, j, j, 1); 123 124 efx_writeo(efx, ®, address); 125 efx_reado(efx, &buf, address); 126 127 if (efx_masked_compare_oword(®, &buf, &mask)) 128 goto fail; 129 130 /* Test this testable bit can be cleared in isolation */ 131 EFX_OR_OWORD(reg, original, mask); 132 EFX_SET_OWORD32(reg, j, j, 0); 133 134 efx_writeo(efx, ®, address); 135 efx_reado(efx, &buf, address); 136 137 if (efx_masked_compare_oword(®, &buf, &mask)) 138 goto fail; 139 } 140 141 efx_writeo(efx, &original, address); 142 } 143 144 return 0; 145 146fail: 147 netif_err(efx, hw, efx->net_dev, 148 "wrote "EFX_OWORD_FMT" read "EFX_OWORD_FMT 149 " at address 0x%x mask "EFX_OWORD_FMT"\n", EFX_OWORD_VAL(reg), 150 EFX_OWORD_VAL(buf), address, EFX_OWORD_VAL(mask)); 151 return -EIO; 152} 153 154/************************************************************************** 155 * 156 * Special buffer handling 157 * Special buffers are used for event queues and the TX and RX 158 * descriptor rings. 159 * 160 *************************************************************************/ 161 162/* 163 * Initialise a special buffer 164 * 165 * This will define a buffer (previously allocated via 166 * efx_alloc_special_buffer()) in the buffer table, allowing 167 * it to be used for event queues, descriptor rings etc. 168 */ 169static void 170efx_init_special_buffer(struct efx_nic *efx, struct efx_special_buffer *buffer) 171{ 172 efx_qword_t buf_desc; 173 unsigned int index; 174 dma_addr_t dma_addr; 175 int i; 176 177 EFX_BUG_ON_PARANOID(!buffer->buf.addr); 178 179 /* Write buffer descriptors to NIC */ 180 for (i = 0; i < buffer->entries; i++) { 181 index = buffer->index + i; 182 dma_addr = buffer->buf.dma_addr + (i * EFX_BUF_SIZE); 183 netif_dbg(efx, probe, efx->net_dev, 184 "mapping special buffer %d at %llx\n", 185 index, (unsigned long long)dma_addr); 186 EFX_POPULATE_QWORD_3(buf_desc, 187 FRF_AZ_BUF_ADR_REGION, 0, 188 FRF_AZ_BUF_ADR_FBUF, dma_addr >> 12, 189 FRF_AZ_BUF_OWNER_ID_FBUF, 0); 190 efx_write_buf_tbl(efx, &buf_desc, index); 191 } 192} 193 194/* Unmaps a buffer and clears the buffer table entries */ 195static void 196efx_fini_special_buffer(struct efx_nic *efx, struct efx_special_buffer *buffer) 197{ 198 efx_oword_t buf_tbl_upd; 199 unsigned int start = buffer->index; 200 unsigned int end = (buffer->index + buffer->entries - 1); 201 202 if (!buffer->entries) 203 return; 204 205 netif_dbg(efx, hw, efx->net_dev, "unmapping special buffers %d-%d\n", 206 buffer->index, buffer->index + buffer->entries - 1); 207 208 EFX_POPULATE_OWORD_4(buf_tbl_upd, 209 FRF_AZ_BUF_UPD_CMD, 0, 210 FRF_AZ_BUF_CLR_CMD, 1, 211 FRF_AZ_BUF_CLR_END_ID, end, 212 FRF_AZ_BUF_CLR_START_ID, start); 213 efx_writeo(efx, &buf_tbl_upd, FR_AZ_BUF_TBL_UPD); 214} 215 216/* 217 * Allocate a new special buffer 218 * 219 * This allocates memory for a new buffer, clears it and allocates a 220 * new buffer ID range. It does not write into the buffer table. 221 * 222 * This call will allocate 4KB buffers, since 8KB buffers can't be 223 * used for event queues and descriptor rings. 224 */ 225static int efx_alloc_special_buffer(struct efx_nic *efx, 226 struct efx_special_buffer *buffer, 227 unsigned int len) 228{ 229#ifdef CONFIG_SFC_SRIOV 230 struct siena_nic_data *nic_data = efx->nic_data; 231#endif 232 len = ALIGN(len, EFX_BUF_SIZE); 233 234 if (efx_nic_alloc_buffer(efx, &buffer->buf, len, GFP_KERNEL)) 235 return -ENOMEM; 236 buffer->entries = len / EFX_BUF_SIZE; 237 BUG_ON(buffer->buf.dma_addr & (EFX_BUF_SIZE - 1)); 238 239 /* Select new buffer ID */ 240 buffer->index = efx->next_buffer_table; 241 efx->next_buffer_table += buffer->entries; 242#ifdef CONFIG_SFC_SRIOV 243 BUG_ON(efx_siena_sriov_enabled(efx) && 244 nic_data->vf_buftbl_base < efx->next_buffer_table); 245#endif 246 247 netif_dbg(efx, probe, efx->net_dev, 248 "allocating special buffers %d-%d at %llx+%x " 249 "(virt %p phys %llx)\n", buffer->index, 250 buffer->index + buffer->entries - 1, 251 (u64)buffer->buf.dma_addr, len, 252 buffer->buf.addr, (u64)virt_to_phys(buffer->buf.addr)); 253 254 return 0; 255} 256 257static void 258efx_free_special_buffer(struct efx_nic *efx, struct efx_special_buffer *buffer) 259{ 260 if (!buffer->buf.addr) 261 return; 262 263 netif_dbg(efx, hw, efx->net_dev, 264 "deallocating special buffers %d-%d at %llx+%x " 265 "(virt %p phys %llx)\n", buffer->index, 266 buffer->index + buffer->entries - 1, 267 (u64)buffer->buf.dma_addr, buffer->buf.len, 268 buffer->buf.addr, (u64)virt_to_phys(buffer->buf.addr)); 269 270 efx_nic_free_buffer(efx, &buffer->buf); 271 buffer->entries = 0; 272} 273 274/************************************************************************** 275 * 276 * TX path 277 * 278 **************************************************************************/ 279 280/* This writes to the TX_DESC_WPTR; write pointer for TX descriptor ring */ 281static inline void efx_farch_notify_tx_desc(struct efx_tx_queue *tx_queue) 282{ 283 unsigned write_ptr; 284 efx_dword_t reg; 285 286 write_ptr = tx_queue->write_count & tx_queue->ptr_mask; 287 EFX_POPULATE_DWORD_1(reg, FRF_AZ_TX_DESC_WPTR_DWORD, write_ptr); 288 efx_writed_page(tx_queue->efx, ®, 289 FR_AZ_TX_DESC_UPD_DWORD_P0, tx_queue->queue); 290} 291 292/* Write pointer and first descriptor for TX descriptor ring */ 293static inline void efx_farch_push_tx_desc(struct efx_tx_queue *tx_queue, 294 const efx_qword_t *txd) 295{ 296 unsigned write_ptr; 297 efx_oword_t reg; 298 299 BUILD_BUG_ON(FRF_AZ_TX_DESC_LBN != 0); 300 BUILD_BUG_ON(FR_AA_TX_DESC_UPD_KER != FR_BZ_TX_DESC_UPD_P0); 301 302 write_ptr = tx_queue->write_count & tx_queue->ptr_mask; 303 EFX_POPULATE_OWORD_2(reg, FRF_AZ_TX_DESC_PUSH_CMD, true, 304 FRF_AZ_TX_DESC_WPTR, write_ptr); 305 reg.qword[0] = *txd; 306 efx_writeo_page(tx_queue->efx, ®, 307 FR_BZ_TX_DESC_UPD_P0, tx_queue->queue); 308} 309 310 311/* For each entry inserted into the software descriptor ring, create a 312 * descriptor in the hardware TX descriptor ring (in host memory), and 313 * write a doorbell. 314 */ 315void efx_farch_tx_write(struct efx_tx_queue *tx_queue) 316{ 317 struct efx_tx_buffer *buffer; 318 efx_qword_t *txd; 319 unsigned write_ptr; 320 unsigned old_write_count = tx_queue->write_count; 321 322 tx_queue->xmit_more_available = false; 323 if (unlikely(tx_queue->write_count == tx_queue->insert_count)) 324 return; 325 326 do { 327 write_ptr = tx_queue->write_count & tx_queue->ptr_mask; 328 buffer = &tx_queue->buffer[write_ptr]; 329 txd = efx_tx_desc(tx_queue, write_ptr); 330 ++tx_queue->write_count; 331 332 EFX_BUG_ON_PARANOID(buffer->flags & EFX_TX_BUF_OPTION); 333 334 /* Create TX descriptor ring entry */ 335 BUILD_BUG_ON(EFX_TX_BUF_CONT != 1); 336 EFX_POPULATE_QWORD_4(*txd, 337 FSF_AZ_TX_KER_CONT, 338 buffer->flags & EFX_TX_BUF_CONT, 339 FSF_AZ_TX_KER_BYTE_COUNT, buffer->len, 340 FSF_AZ_TX_KER_BUF_REGION, 0, 341 FSF_AZ_TX_KER_BUF_ADDR, buffer->dma_addr); 342 } while (tx_queue->write_count != tx_queue->insert_count); 343 344 wmb(); /* Ensure descriptors are written before they are fetched */ 345 346 if (efx_nic_may_push_tx_desc(tx_queue, old_write_count)) { 347 txd = efx_tx_desc(tx_queue, 348 old_write_count & tx_queue->ptr_mask); 349 efx_farch_push_tx_desc(tx_queue, txd); 350 ++tx_queue->pushes; 351 } else { 352 efx_farch_notify_tx_desc(tx_queue); 353 } 354} 355 356/* Allocate hardware resources for a TX queue */ 357int efx_farch_tx_probe(struct efx_tx_queue *tx_queue) 358{ 359 struct efx_nic *efx = tx_queue->efx; 360 unsigned entries; 361 362 entries = tx_queue->ptr_mask + 1; 363 return efx_alloc_special_buffer(efx, &tx_queue->txd, 364 entries * sizeof(efx_qword_t)); 365} 366 367void efx_farch_tx_init(struct efx_tx_queue *tx_queue) 368{ 369 struct efx_nic *efx = tx_queue->efx; 370 efx_oword_t reg; 371 372 /* Pin TX descriptor ring */ 373 efx_init_special_buffer(efx, &tx_queue->txd); 374 375 /* Push TX descriptor ring to card */ 376 EFX_POPULATE_OWORD_10(reg, 377 FRF_AZ_TX_DESCQ_EN, 1, 378 FRF_AZ_TX_ISCSI_DDIG_EN, 0, 379 FRF_AZ_TX_ISCSI_HDIG_EN, 0, 380 FRF_AZ_TX_DESCQ_BUF_BASE_ID, tx_queue->txd.index, 381 FRF_AZ_TX_DESCQ_EVQ_ID, 382 tx_queue->channel->channel, 383 FRF_AZ_TX_DESCQ_OWNER_ID, 0, 384 FRF_AZ_TX_DESCQ_LABEL, tx_queue->queue, 385 FRF_AZ_TX_DESCQ_SIZE, 386 __ffs(tx_queue->txd.entries), 387 FRF_AZ_TX_DESCQ_TYPE, 0, 388 FRF_BZ_TX_NON_IP_DROP_DIS, 1); 389 390 if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) { 391 int csum = tx_queue->queue & EFX_TXQ_TYPE_OFFLOAD; 392 EFX_SET_OWORD_FIELD(reg, FRF_BZ_TX_IP_CHKSM_DIS, !csum); 393 EFX_SET_OWORD_FIELD(reg, FRF_BZ_TX_TCP_CHKSM_DIS, 394 !csum); 395 } 396 397 efx_writeo_table(efx, ®, efx->type->txd_ptr_tbl_base, 398 tx_queue->queue); 399 400 if (efx_nic_rev(efx) < EFX_REV_FALCON_B0) { 401 /* Only 128 bits in this register */ 402 BUILD_BUG_ON(EFX_MAX_TX_QUEUES > 128); 403 404 efx_reado(efx, ®, FR_AA_TX_CHKSM_CFG); 405 if (tx_queue->queue & EFX_TXQ_TYPE_OFFLOAD) 406 __clear_bit_le(tx_queue->queue, ®); 407 else 408 __set_bit_le(tx_queue->queue, ®); 409 efx_writeo(efx, ®, FR_AA_TX_CHKSM_CFG); 410 } 411 412 if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) { 413 EFX_POPULATE_OWORD_1(reg, 414 FRF_BZ_TX_PACE, 415 (tx_queue->queue & EFX_TXQ_TYPE_HIGHPRI) ? 416 FFE_BZ_TX_PACE_OFF : 417 FFE_BZ_TX_PACE_RESERVED); 418 efx_writeo_table(efx, ®, FR_BZ_TX_PACE_TBL, 419 tx_queue->queue); 420 } 421} 422 423static void efx_farch_flush_tx_queue(struct efx_tx_queue *tx_queue) 424{ 425 struct efx_nic *efx = tx_queue->efx; 426 efx_oword_t tx_flush_descq; 427 428 WARN_ON(atomic_read(&tx_queue->flush_outstanding)); 429 atomic_set(&tx_queue->flush_outstanding, 1); 430 431 EFX_POPULATE_OWORD_2(tx_flush_descq, 432 FRF_AZ_TX_FLUSH_DESCQ_CMD, 1, 433 FRF_AZ_TX_FLUSH_DESCQ, tx_queue->queue); 434 efx_writeo(efx, &tx_flush_descq, FR_AZ_TX_FLUSH_DESCQ); 435} 436 437void efx_farch_tx_fini(struct efx_tx_queue *tx_queue) 438{ 439 struct efx_nic *efx = tx_queue->efx; 440 efx_oword_t tx_desc_ptr; 441 442 /* Remove TX descriptor ring from card */ 443 EFX_ZERO_OWORD(tx_desc_ptr); 444 efx_writeo_table(efx, &tx_desc_ptr, efx->type->txd_ptr_tbl_base, 445 tx_queue->queue); 446 447 /* Unpin TX descriptor ring */ 448 efx_fini_special_buffer(efx, &tx_queue->txd); 449} 450 451/* Free buffers backing TX queue */ 452void efx_farch_tx_remove(struct efx_tx_queue *tx_queue) 453{ 454 efx_free_special_buffer(tx_queue->efx, &tx_queue->txd); 455} 456 457/************************************************************************** 458 * 459 * RX path 460 * 461 **************************************************************************/ 462 463/* This creates an entry in the RX descriptor queue */ 464static inline void 465efx_farch_build_rx_desc(struct efx_rx_queue *rx_queue, unsigned index) 466{ 467 struct efx_rx_buffer *rx_buf; 468 efx_qword_t *rxd; 469 470 rxd = efx_rx_desc(rx_queue, index); 471 rx_buf = efx_rx_buffer(rx_queue, index); 472 EFX_POPULATE_QWORD_3(*rxd, 473 FSF_AZ_RX_KER_BUF_SIZE, 474 rx_buf->len - 475 rx_queue->efx->type->rx_buffer_padding, 476 FSF_AZ_RX_KER_BUF_REGION, 0, 477 FSF_AZ_RX_KER_BUF_ADDR, rx_buf->dma_addr); 478} 479 480/* This writes to the RX_DESC_WPTR register for the specified receive 481 * descriptor ring. 482 */ 483void efx_farch_rx_write(struct efx_rx_queue *rx_queue) 484{ 485 struct efx_nic *efx = rx_queue->efx; 486 efx_dword_t reg; 487 unsigned write_ptr; 488 489 while (rx_queue->notified_count != rx_queue->added_count) { 490 efx_farch_build_rx_desc( 491 rx_queue, 492 rx_queue->notified_count & rx_queue->ptr_mask); 493 ++rx_queue->notified_count; 494 } 495 496 wmb(); 497 write_ptr = rx_queue->added_count & rx_queue->ptr_mask; 498 EFX_POPULATE_DWORD_1(reg, FRF_AZ_RX_DESC_WPTR_DWORD, write_ptr); 499 efx_writed_page(efx, ®, FR_AZ_RX_DESC_UPD_DWORD_P0, 500 efx_rx_queue_index(rx_queue)); 501} 502 503int efx_farch_rx_probe(struct efx_rx_queue *rx_queue) 504{ 505 struct efx_nic *efx = rx_queue->efx; 506 unsigned entries; 507 508 entries = rx_queue->ptr_mask + 1; 509 return efx_alloc_special_buffer(efx, &rx_queue->rxd, 510 entries * sizeof(efx_qword_t)); 511} 512 513void efx_farch_rx_init(struct efx_rx_queue *rx_queue) 514{ 515 efx_oword_t rx_desc_ptr; 516 struct efx_nic *efx = rx_queue->efx; 517 bool is_b0 = efx_nic_rev(efx) >= EFX_REV_FALCON_B0; 518 bool iscsi_digest_en = is_b0; 519 bool jumbo_en; 520 521 /* For kernel-mode queues in Falcon A1, the JUMBO flag enables 522 * DMA to continue after a PCIe page boundary (and scattering 523 * is not possible). In Falcon B0 and Siena, it enables 524 * scatter. 525 */ 526 jumbo_en = !is_b0 || efx->rx_scatter; 527 528 netif_dbg(efx, hw, efx->net_dev, 529 "RX queue %d ring in special buffers %d-%d\n", 530 efx_rx_queue_index(rx_queue), rx_queue->rxd.index, 531 rx_queue->rxd.index + rx_queue->rxd.entries - 1); 532 533 rx_queue->scatter_n = 0; 534 535 /* Pin RX descriptor ring */ 536 efx_init_special_buffer(efx, &rx_queue->rxd); 537 538 /* Push RX descriptor ring to card */ 539 EFX_POPULATE_OWORD_10(rx_desc_ptr, 540 FRF_AZ_RX_ISCSI_DDIG_EN, iscsi_digest_en, 541 FRF_AZ_RX_ISCSI_HDIG_EN, iscsi_digest_en, 542 FRF_AZ_RX_DESCQ_BUF_BASE_ID, rx_queue->rxd.index, 543 FRF_AZ_RX_DESCQ_EVQ_ID, 544 efx_rx_queue_channel(rx_queue)->channel, 545 FRF_AZ_RX_DESCQ_OWNER_ID, 0, 546 FRF_AZ_RX_DESCQ_LABEL, 547 efx_rx_queue_index(rx_queue), 548 FRF_AZ_RX_DESCQ_SIZE, 549 __ffs(rx_queue->rxd.entries), 550 FRF_AZ_RX_DESCQ_TYPE, 0 /* kernel queue */ , 551 FRF_AZ_RX_DESCQ_JUMBO, jumbo_en, 552 FRF_AZ_RX_DESCQ_EN, 1); 553 efx_writeo_table(efx, &rx_desc_ptr, efx->type->rxd_ptr_tbl_base, 554 efx_rx_queue_index(rx_queue)); 555} 556 557static void efx_farch_flush_rx_queue(struct efx_rx_queue *rx_queue) 558{ 559 struct efx_nic *efx = rx_queue->efx; 560 efx_oword_t rx_flush_descq; 561 562 EFX_POPULATE_OWORD_2(rx_flush_descq, 563 FRF_AZ_RX_FLUSH_DESCQ_CMD, 1, 564 FRF_AZ_RX_FLUSH_DESCQ, 565 efx_rx_queue_index(rx_queue)); 566 efx_writeo(efx, &rx_flush_descq, FR_AZ_RX_FLUSH_DESCQ); 567} 568 569void efx_farch_rx_fini(struct efx_rx_queue *rx_queue) 570{ 571 efx_oword_t rx_desc_ptr; 572 struct efx_nic *efx = rx_queue->efx; 573 574 /* Remove RX descriptor ring from card */ 575 EFX_ZERO_OWORD(rx_desc_ptr); 576 efx_writeo_table(efx, &rx_desc_ptr, efx->type->rxd_ptr_tbl_base, 577 efx_rx_queue_index(rx_queue)); 578 579 /* Unpin RX descriptor ring */ 580 efx_fini_special_buffer(efx, &rx_queue->rxd); 581} 582 583/* Free buffers backing RX queue */ 584void efx_farch_rx_remove(struct efx_rx_queue *rx_queue) 585{ 586 efx_free_special_buffer(rx_queue->efx, &rx_queue->rxd); 587} 588 589/************************************************************************** 590 * 591 * Flush handling 592 * 593 **************************************************************************/ 594 595/* efx_farch_flush_queues() must be woken up when all flushes are completed, 596 * or more RX flushes can be kicked off. 597 */ 598static bool efx_farch_flush_wake(struct efx_nic *efx) 599{ 600 /* Ensure that all updates are visible to efx_farch_flush_queues() */ 601 smp_mb(); 602 603 return (atomic_read(&efx->active_queues) == 0 || 604 (atomic_read(&efx->rxq_flush_outstanding) < EFX_RX_FLUSH_COUNT 605 && atomic_read(&efx->rxq_flush_pending) > 0)); 606} 607 608static bool efx_check_tx_flush_complete(struct efx_nic *efx) 609{ 610 bool i = true; 611 efx_oword_t txd_ptr_tbl; 612 struct efx_channel *channel; 613 struct efx_tx_queue *tx_queue; 614 615 efx_for_each_channel(channel, efx) { 616 efx_for_each_channel_tx_queue(tx_queue, channel) { 617 efx_reado_table(efx, &txd_ptr_tbl, 618 FR_BZ_TX_DESC_PTR_TBL, tx_queue->queue); 619 if (EFX_OWORD_FIELD(txd_ptr_tbl, 620 FRF_AZ_TX_DESCQ_FLUSH) || 621 EFX_OWORD_FIELD(txd_ptr_tbl, 622 FRF_AZ_TX_DESCQ_EN)) { 623 netif_dbg(efx, hw, efx->net_dev, 624 "flush did not complete on TXQ %d\n", 625 tx_queue->queue); 626 i = false; 627 } else if (atomic_cmpxchg(&tx_queue->flush_outstanding, 628 1, 0)) { 629 /* The flush is complete, but we didn't 630 * receive a flush completion event 631 */ 632 netif_dbg(efx, hw, efx->net_dev, 633 "flush complete on TXQ %d, so drain " 634 "the queue\n", tx_queue->queue); 635 /* Don't need to increment active_queues as it 636 * has already been incremented for the queues 637 * which did not drain 638 */ 639 efx_farch_magic_event(channel, 640 EFX_CHANNEL_MAGIC_TX_DRAIN( 641 tx_queue)); 642 } 643 } 644 } 645 646 return i; 647} 648 649/* Flush all the transmit queues, and continue flushing receive queues until 650 * they're all flushed. Wait for the DRAIN events to be received so that there 651 * are no more RX and TX events left on any channel. */ 652static int efx_farch_do_flush(struct efx_nic *efx) 653{ 654 unsigned timeout = msecs_to_jiffies(5000); /* 5s for all flushes and drains */ 655 struct efx_channel *channel; 656 struct efx_rx_queue *rx_queue; 657 struct efx_tx_queue *tx_queue; 658 int rc = 0; 659 660 efx_for_each_channel(channel, efx) { 661 efx_for_each_channel_tx_queue(tx_queue, channel) { 662 efx_farch_flush_tx_queue(tx_queue); 663 } 664 efx_for_each_channel_rx_queue(rx_queue, channel) { 665 rx_queue->flush_pending = true; 666 atomic_inc(&efx->rxq_flush_pending); 667 } 668 } 669 670 while (timeout && atomic_read(&efx->active_queues) > 0) { 671 /* If SRIOV is enabled, then offload receive queue flushing to 672 * the firmware (though we will still have to poll for 673 * completion). If that fails, fall back to the old scheme. 674 */ 675 if (efx_siena_sriov_enabled(efx)) { 676 rc = efx_mcdi_flush_rxqs(efx); 677 if (!rc) 678 goto wait; 679 } 680 681 /* The hardware supports four concurrent rx flushes, each of 682 * which may need to be retried if there is an outstanding 683 * descriptor fetch 684 */ 685 efx_for_each_channel(channel, efx) { 686 efx_for_each_channel_rx_queue(rx_queue, channel) { 687 if (atomic_read(&efx->rxq_flush_outstanding) >= 688 EFX_RX_FLUSH_COUNT) 689 break; 690 691 if (rx_queue->flush_pending) { 692 rx_queue->flush_pending = false; 693 atomic_dec(&efx->rxq_flush_pending); 694 atomic_inc(&efx->rxq_flush_outstanding); 695 efx_farch_flush_rx_queue(rx_queue); 696 } 697 } 698 } 699 700 wait: 701 timeout = wait_event_timeout(efx->flush_wq, 702 efx_farch_flush_wake(efx), 703 timeout); 704 } 705 706 if (atomic_read(&efx->active_queues) && 707 !efx_check_tx_flush_complete(efx)) { 708 netif_err(efx, hw, efx->net_dev, "failed to flush %d queues " 709 "(rx %d+%d)\n", atomic_read(&efx->active_queues), 710 atomic_read(&efx->rxq_flush_outstanding), 711 atomic_read(&efx->rxq_flush_pending)); 712 rc = -ETIMEDOUT; 713 714 atomic_set(&efx->active_queues, 0); 715 atomic_set(&efx->rxq_flush_pending, 0); 716 atomic_set(&efx->rxq_flush_outstanding, 0); 717 } 718 719 return rc; 720} 721 722int efx_farch_fini_dmaq(struct efx_nic *efx) 723{ 724 struct efx_channel *channel; 725 struct efx_tx_queue *tx_queue; 726 struct efx_rx_queue *rx_queue; 727 int rc = 0; 728 729 /* Do not attempt to write to the NIC during EEH recovery */ 730 if (efx->state != STATE_RECOVERY) { 731 /* Only perform flush if DMA is enabled */ 732 if (efx->pci_dev->is_busmaster) { 733 efx->type->prepare_flush(efx); 734 rc = efx_farch_do_flush(efx); 735 efx->type->finish_flush(efx); 736 } 737 738 efx_for_each_channel(channel, efx) { 739 efx_for_each_channel_rx_queue(rx_queue, channel) 740 efx_farch_rx_fini(rx_queue); 741 efx_for_each_channel_tx_queue(tx_queue, channel) 742 efx_farch_tx_fini(tx_queue); 743 } 744 } 745 746 return rc; 747} 748 749/* Reset queue and flush accounting after FLR 750 * 751 * One possible cause of FLR recovery is that DMA may be failing (eg. if bus 752 * mastering was disabled), in which case we don't receive (RXQ) flush 753 * completion events. This means that efx->rxq_flush_outstanding remained at 4 754 * after the FLR; also, efx->active_queues was non-zero (as no flush completion 755 * events were received, and we didn't go through efx_check_tx_flush_complete()) 756 * If we don't fix this up, on the next call to efx_realloc_channels() we won't 757 * flush any RX queues because efx->rxq_flush_outstanding is at the limit of 4 758 * for batched flush requests; and the efx->active_queues gets messed up because 759 * we keep incrementing for the newly initialised queues, but it never went to 760 * zero previously. Then we get a timeout every time we try to restart the 761 * queues, as it doesn't go back to zero when we should be flushing the queues. 762 */ 763void efx_farch_finish_flr(struct efx_nic *efx) 764{ 765 atomic_set(&efx->rxq_flush_pending, 0); 766 atomic_set(&efx->rxq_flush_outstanding, 0); 767 atomic_set(&efx->active_queues, 0); 768} 769 770 771/************************************************************************** 772 * 773 * Event queue processing 774 * Event queues are processed by per-channel tasklets. 775 * 776 **************************************************************************/ 777 778/* Update a channel's event queue's read pointer (RPTR) register 779 * 780 * This writes the EVQ_RPTR_REG register for the specified channel's 781 * event queue. 782 */ 783void efx_farch_ev_read_ack(struct efx_channel *channel) 784{ 785 efx_dword_t reg; 786 struct efx_nic *efx = channel->efx; 787 788 EFX_POPULATE_DWORD_1(reg, FRF_AZ_EVQ_RPTR, 789 channel->eventq_read_ptr & channel->eventq_mask); 790 791 /* For Falcon A1, EVQ_RPTR_KER is documented as having a step size 792 * of 4 bytes, but it is really 16 bytes just like later revisions. 793 */ 794 efx_writed(efx, ®, 795 efx->type->evq_rptr_tbl_base + 796 FR_BZ_EVQ_RPTR_STEP * channel->channel); 797} 798 799/* Use HW to insert a SW defined event */ 800void efx_farch_generate_event(struct efx_nic *efx, unsigned int evq, 801 efx_qword_t *event) 802{ 803 efx_oword_t drv_ev_reg; 804 805 BUILD_BUG_ON(FRF_AZ_DRV_EV_DATA_LBN != 0 || 806 FRF_AZ_DRV_EV_DATA_WIDTH != 64); 807 drv_ev_reg.u32[0] = event->u32[0]; 808 drv_ev_reg.u32[1] = event->u32[1]; 809 drv_ev_reg.u32[2] = 0; 810 drv_ev_reg.u32[3] = 0; 811 EFX_SET_OWORD_FIELD(drv_ev_reg, FRF_AZ_DRV_EV_QID, evq); 812 efx_writeo(efx, &drv_ev_reg, FR_AZ_DRV_EV); 813} 814 815static void efx_farch_magic_event(struct efx_channel *channel, u32 magic) 816{ 817 efx_qword_t event; 818 819 EFX_POPULATE_QWORD_2(event, FSF_AZ_EV_CODE, 820 FSE_AZ_EV_CODE_DRV_GEN_EV, 821 FSF_AZ_DRV_GEN_EV_MAGIC, magic); 822 efx_farch_generate_event(channel->efx, channel->channel, &event); 823} 824 825/* Handle a transmit completion event 826 * 827 * The NIC batches TX completion events; the message we receive is of 828 * the form "complete all TX events up to this index". 829 */ 830static int 831efx_farch_handle_tx_event(struct efx_channel *channel, efx_qword_t *event) 832{ 833 unsigned int tx_ev_desc_ptr; 834 unsigned int tx_ev_q_label; 835 struct efx_tx_queue *tx_queue; 836 struct efx_nic *efx = channel->efx; 837 int tx_packets = 0; 838 839 if (unlikely(ACCESS_ONCE(efx->reset_pending))) 840 return 0; 841 842 if (likely(EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_COMP))) { 843 /* Transmit completion */ 844 tx_ev_desc_ptr = EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_DESC_PTR); 845 tx_ev_q_label = EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_Q_LABEL); 846 tx_queue = efx_channel_get_tx_queue( 847 channel, tx_ev_q_label % EFX_TXQ_TYPES); 848 tx_packets = ((tx_ev_desc_ptr - tx_queue->read_count) & 849 tx_queue->ptr_mask); 850 efx_xmit_done(tx_queue, tx_ev_desc_ptr); 851 } else if (EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_WQ_FF_FULL)) { 852 /* Rewrite the FIFO write pointer */ 853 tx_ev_q_label = EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_Q_LABEL); 854 tx_queue = efx_channel_get_tx_queue( 855 channel, tx_ev_q_label % EFX_TXQ_TYPES); 856 857 netif_tx_lock(efx->net_dev); 858 efx_farch_notify_tx_desc(tx_queue); 859 netif_tx_unlock(efx->net_dev); 860 } else if (EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_PKT_ERR)) { 861 efx_schedule_reset(efx, RESET_TYPE_DMA_ERROR); 862 } else { 863 netif_err(efx, tx_err, efx->net_dev, 864 "channel %d unexpected TX event " 865 EFX_QWORD_FMT"\n", channel->channel, 866 EFX_QWORD_VAL(*event)); 867 } 868 869 return tx_packets; 870} 871 872/* Detect errors included in the rx_evt_pkt_ok bit. */ 873static u16 efx_farch_handle_rx_not_ok(struct efx_rx_queue *rx_queue, 874 const efx_qword_t *event) 875{ 876 struct efx_channel *channel = efx_rx_queue_channel(rx_queue); 877 struct efx_nic *efx = rx_queue->efx; 878 bool rx_ev_buf_owner_id_err, rx_ev_ip_hdr_chksum_err; 879 bool rx_ev_tcp_udp_chksum_err, rx_ev_eth_crc_err; 880 bool rx_ev_frm_trunc, rx_ev_drib_nib, rx_ev_tobe_disc; 881 bool rx_ev_other_err, rx_ev_pause_frm; 882 bool rx_ev_hdr_type, rx_ev_mcast_pkt; 883 unsigned rx_ev_pkt_type; 884 885 rx_ev_hdr_type = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_HDR_TYPE); 886 rx_ev_mcast_pkt = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_MCAST_PKT); 887 rx_ev_tobe_disc = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_TOBE_DISC); 888 rx_ev_pkt_type = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_PKT_TYPE); 889 rx_ev_buf_owner_id_err = EFX_QWORD_FIELD(*event, 890 FSF_AZ_RX_EV_BUF_OWNER_ID_ERR); 891 rx_ev_ip_hdr_chksum_err = EFX_QWORD_FIELD(*event, 892 FSF_AZ_RX_EV_IP_HDR_CHKSUM_ERR); 893 rx_ev_tcp_udp_chksum_err = EFX_QWORD_FIELD(*event, 894 FSF_AZ_RX_EV_TCP_UDP_CHKSUM_ERR); 895 rx_ev_eth_crc_err = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_ETH_CRC_ERR); 896 rx_ev_frm_trunc = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_FRM_TRUNC); 897 rx_ev_drib_nib = ((efx_nic_rev(efx) >= EFX_REV_FALCON_B0) ? 898 0 : EFX_QWORD_FIELD(*event, FSF_AA_RX_EV_DRIB_NIB)); 899 rx_ev_pause_frm = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_PAUSE_FRM_ERR); 900 901 /* Every error apart from tobe_disc and pause_frm */ 902 rx_ev_other_err = (rx_ev_drib_nib | rx_ev_tcp_udp_chksum_err | 903 rx_ev_buf_owner_id_err | rx_ev_eth_crc_err | 904 rx_ev_frm_trunc | rx_ev_ip_hdr_chksum_err); 905 906 /* Count errors that are not in MAC stats. Ignore expected 907 * checksum errors during self-test. */ 908 if (rx_ev_frm_trunc) 909 ++channel->n_rx_frm_trunc; 910 else if (rx_ev_tobe_disc) 911 ++channel->n_rx_tobe_disc; 912 else if (!efx->loopback_selftest) { 913 if (rx_ev_ip_hdr_chksum_err) 914 ++channel->n_rx_ip_hdr_chksum_err; 915 else if (rx_ev_tcp_udp_chksum_err) 916 ++channel->n_rx_tcp_udp_chksum_err; 917 } 918 919 /* TOBE_DISC is expected on unicast mismatches; don't print out an 920 * error message. FRM_TRUNC indicates RXDP dropped the packet due 921 * to a FIFO overflow. 922 */ 923#ifdef DEBUG 924 if (rx_ev_other_err && net_ratelimit()) { 925 netif_dbg(efx, rx_err, efx->net_dev, 926 " RX queue %d unexpected RX event " 927 EFX_QWORD_FMT "%s%s%s%s%s%s%s%s\n", 928 efx_rx_queue_index(rx_queue), EFX_QWORD_VAL(*event), 929 rx_ev_buf_owner_id_err ? " [OWNER_ID_ERR]" : "", 930 rx_ev_ip_hdr_chksum_err ? 931 " [IP_HDR_CHKSUM_ERR]" : "", 932 rx_ev_tcp_udp_chksum_err ? 933 " [TCP_UDP_CHKSUM_ERR]" : "", 934 rx_ev_eth_crc_err ? " [ETH_CRC_ERR]" : "", 935 rx_ev_frm_trunc ? " [FRM_TRUNC]" : "", 936 rx_ev_drib_nib ? " [DRIB_NIB]" : "", 937 rx_ev_tobe_disc ? " [TOBE_DISC]" : "", 938 rx_ev_pause_frm ? " [PAUSE]" : ""); 939 } 940#endif 941 942 /* The frame must be discarded if any of these are true. */ 943 return (rx_ev_eth_crc_err | rx_ev_frm_trunc | rx_ev_drib_nib | 944 rx_ev_tobe_disc | rx_ev_pause_frm) ? 945 EFX_RX_PKT_DISCARD : 0; 946} 947 948/* Handle receive events that are not in-order. Return true if this 949 * can be handled as a partial packet discard, false if it's more 950 * serious. 951 */ 952static bool 953efx_farch_handle_rx_bad_index(struct efx_rx_queue *rx_queue, unsigned index) 954{ 955 struct efx_channel *channel = efx_rx_queue_channel(rx_queue); 956 struct efx_nic *efx = rx_queue->efx; 957 unsigned expected, dropped; 958 959 if (rx_queue->scatter_n && 960 index == ((rx_queue->removed_count + rx_queue->scatter_n - 1) & 961 rx_queue->ptr_mask)) { 962 ++channel->n_rx_nodesc_trunc; 963 return true; 964 } 965 966 expected = rx_queue->removed_count & rx_queue->ptr_mask; 967 dropped = (index - expected) & rx_queue->ptr_mask; 968 netif_info(efx, rx_err, efx->net_dev, 969 "dropped %d events (index=%d expected=%d)\n", 970 dropped, index, expected); 971 972 efx_schedule_reset(efx, EFX_WORKAROUND_5676(efx) ? 973 RESET_TYPE_RX_RECOVERY : RESET_TYPE_DISABLE); 974 return false; 975} 976 977/* Handle a packet received event 978 * 979 * The NIC gives a "discard" flag if it's a unicast packet with the 980 * wrong destination address 981 * Also "is multicast" and "matches multicast filter" flags can be used to 982 * discard non-matching multicast packets. 983 */ 984static void 985efx_farch_handle_rx_event(struct efx_channel *channel, const efx_qword_t *event) 986{ 987 unsigned int rx_ev_desc_ptr, rx_ev_byte_cnt; 988 unsigned int rx_ev_hdr_type, rx_ev_mcast_pkt; 989 unsigned expected_ptr; 990 bool rx_ev_pkt_ok, rx_ev_sop, rx_ev_cont; 991 u16 flags; 992 struct efx_rx_queue *rx_queue; 993 struct efx_nic *efx = channel->efx; 994 995 if (unlikely(ACCESS_ONCE(efx->reset_pending))) 996 return; 997 998 rx_ev_cont = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_JUMBO_CONT); 999 rx_ev_sop = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_SOP); 1000 WARN_ON(EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_Q_LABEL) != 1001 channel->channel); 1002 1003 rx_queue = efx_channel_get_rx_queue(channel); 1004 1005 rx_ev_desc_ptr = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_DESC_PTR); 1006 expected_ptr = ((rx_queue->removed_count + rx_queue->scatter_n) & 1007 rx_queue->ptr_mask); 1008 1009 /* Check for partial drops and other errors */ 1010 if (unlikely(rx_ev_desc_ptr != expected_ptr) || 1011 unlikely(rx_ev_sop != (rx_queue->scatter_n == 0))) { 1012 if (rx_ev_desc_ptr != expected_ptr && 1013 !efx_farch_handle_rx_bad_index(rx_queue, rx_ev_desc_ptr)) 1014 return; 1015 1016 /* Discard all pending fragments */ 1017 if (rx_queue->scatter_n) { 1018 efx_rx_packet( 1019 rx_queue, 1020 rx_queue->removed_count & rx_queue->ptr_mask, 1021 rx_queue->scatter_n, 0, EFX_RX_PKT_DISCARD); 1022 rx_queue->removed_count += rx_queue->scatter_n; 1023 rx_queue->scatter_n = 0; 1024 } 1025 1026 /* Return if there is no new fragment */ 1027 if (rx_ev_desc_ptr != expected_ptr) 1028 return; 1029 1030 /* Discard new fragment if not SOP */ 1031 if (!rx_ev_sop) { 1032 efx_rx_packet( 1033 rx_queue, 1034 rx_queue->removed_count & rx_queue->ptr_mask, 1035 1, 0, EFX_RX_PKT_DISCARD); 1036 ++rx_queue->removed_count; 1037 return; 1038 } 1039 } 1040 1041 ++rx_queue->scatter_n; 1042 if (rx_ev_cont) 1043 return; 1044 1045 rx_ev_byte_cnt = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_BYTE_CNT); 1046 rx_ev_pkt_ok = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_PKT_OK); 1047 rx_ev_hdr_type = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_HDR_TYPE); 1048 1049 if (likely(rx_ev_pkt_ok)) { 1050 /* If packet is marked as OK then we can rely on the 1051 * hardware checksum and classification. 1052 */ 1053 flags = 0; 1054 switch (rx_ev_hdr_type) { 1055 case FSE_CZ_RX_EV_HDR_TYPE_IPV4V6_TCP: 1056 flags |= EFX_RX_PKT_TCP; 1057 /* fall through */ 1058 case FSE_CZ_RX_EV_HDR_TYPE_IPV4V6_UDP: 1059 flags |= EFX_RX_PKT_CSUMMED; 1060 /* fall through */ 1061 case FSE_CZ_RX_EV_HDR_TYPE_IPV4V6_OTHER: 1062 case FSE_AZ_RX_EV_HDR_TYPE_OTHER: 1063 break; 1064 } 1065 } else { 1066 flags = efx_farch_handle_rx_not_ok(rx_queue, event); 1067 } 1068 1069 /* Detect multicast packets that didn't match the filter */ 1070 rx_ev_mcast_pkt = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_MCAST_PKT); 1071 if (rx_ev_mcast_pkt) { 1072 unsigned int rx_ev_mcast_hash_match = 1073 EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_MCAST_HASH_MATCH); 1074 1075 if (unlikely(!rx_ev_mcast_hash_match)) { 1076 ++channel->n_rx_mcast_mismatch; 1077 flags |= EFX_RX_PKT_DISCARD; 1078 } 1079 } 1080 1081 channel->irq_mod_score += 2; 1082 1083 /* Handle received packet */ 1084 efx_rx_packet(rx_queue, 1085 rx_queue->removed_count & rx_queue->ptr_mask, 1086 rx_queue->scatter_n, rx_ev_byte_cnt, flags); 1087 rx_queue->removed_count += rx_queue->scatter_n; 1088 rx_queue->scatter_n = 0; 1089} 1090 1091/* If this flush done event corresponds to a &struct efx_tx_queue, then 1092 * send an %EFX_CHANNEL_MAGIC_TX_DRAIN event to drain the event queue 1093 * of all transmit completions. 1094 */ 1095static void 1096efx_farch_handle_tx_flush_done(struct efx_nic *efx, efx_qword_t *event) 1097{ 1098 struct efx_tx_queue *tx_queue; 1099 int qid; 1100 1101 qid = EFX_QWORD_FIELD(*event, FSF_AZ_DRIVER_EV_SUBDATA); 1102 if (qid < EFX_TXQ_TYPES * efx->n_tx_channels) { 1103 tx_queue = efx_get_tx_queue(efx, qid / EFX_TXQ_TYPES, 1104 qid % EFX_TXQ_TYPES); 1105 if (atomic_cmpxchg(&tx_queue->flush_outstanding, 1, 0)) { 1106 efx_farch_magic_event(tx_queue->channel, 1107 EFX_CHANNEL_MAGIC_TX_DRAIN(tx_queue)); 1108 } 1109 } 1110} 1111 1112/* If this flush done event corresponds to a &struct efx_rx_queue: If the flush 1113 * was successful then send an %EFX_CHANNEL_MAGIC_RX_DRAIN, otherwise add 1114 * the RX queue back to the mask of RX queues in need of flushing. 1115 */ 1116static void 1117efx_farch_handle_rx_flush_done(struct efx_nic *efx, efx_qword_t *event) 1118{ 1119 struct efx_channel *channel; 1120 struct efx_rx_queue *rx_queue; 1121 int qid; 1122 bool failed; 1123 1124 qid = EFX_QWORD_FIELD(*event, FSF_AZ_DRIVER_EV_RX_DESCQ_ID); 1125 failed = EFX_QWORD_FIELD(*event, FSF_AZ_DRIVER_EV_RX_FLUSH_FAIL); 1126 if (qid >= efx->n_channels) 1127 return; 1128 channel = efx_get_channel(efx, qid); 1129 if (!efx_channel_has_rx_queue(channel)) 1130 return; 1131 rx_queue = efx_channel_get_rx_queue(channel); 1132 1133 if (failed) { 1134 netif_info(efx, hw, efx->net_dev, 1135 "RXQ %d flush retry\n", qid); 1136 rx_queue->flush_pending = true; 1137 atomic_inc(&efx->rxq_flush_pending); 1138 } else { 1139 efx_farch_magic_event(efx_rx_queue_channel(rx_queue), 1140 EFX_CHANNEL_MAGIC_RX_DRAIN(rx_queue)); 1141 } 1142 atomic_dec(&efx->rxq_flush_outstanding); 1143 if (efx_farch_flush_wake(efx)) 1144 wake_up(&efx->flush_wq); 1145} 1146 1147static void 1148efx_farch_handle_drain_event(struct efx_channel *channel) 1149{ 1150 struct efx_nic *efx = channel->efx; 1151 1152 WARN_ON(atomic_read(&efx->active_queues) == 0); 1153 atomic_dec(&efx->active_queues); 1154 if (efx_farch_flush_wake(efx)) 1155 wake_up(&efx->flush_wq); 1156} 1157 1158static void efx_farch_handle_generated_event(struct efx_channel *channel, 1159 efx_qword_t *event) 1160{ 1161 struct efx_nic *efx = channel->efx; 1162 struct efx_rx_queue *rx_queue = 1163 efx_channel_has_rx_queue(channel) ? 1164 efx_channel_get_rx_queue(channel) : NULL; 1165 unsigned magic, code; 1166 1167 magic = EFX_QWORD_FIELD(*event, FSF_AZ_DRV_GEN_EV_MAGIC); 1168 code = _EFX_CHANNEL_MAGIC_CODE(magic); 1169 1170 if (magic == EFX_CHANNEL_MAGIC_TEST(channel)) { 1171 channel->event_test_cpu = raw_smp_processor_id(); 1172 } else if (rx_queue && magic == EFX_CHANNEL_MAGIC_FILL(rx_queue)) { 1173 /* The queue must be empty, so we won't receive any rx 1174 * events, so efx_process_channel() won't refill the 1175 * queue. Refill it here */ 1176 efx_fast_push_rx_descriptors(rx_queue, true); 1177 } else if (rx_queue && magic == EFX_CHANNEL_MAGIC_RX_DRAIN(rx_queue)) { 1178 efx_farch_handle_drain_event(channel); 1179 } else if (code == _EFX_CHANNEL_MAGIC_TX_DRAIN) { 1180 efx_farch_handle_drain_event(channel); 1181 } else { 1182 netif_dbg(efx, hw, efx->net_dev, "channel %d received " 1183 "generated event "EFX_QWORD_FMT"\n", 1184 channel->channel, EFX_QWORD_VAL(*event)); 1185 } 1186} 1187 1188static void 1189efx_farch_handle_driver_event(struct efx_channel *channel, efx_qword_t *event) 1190{ 1191 struct efx_nic *efx = channel->efx; 1192 unsigned int ev_sub_code; 1193 unsigned int ev_sub_data; 1194 1195 ev_sub_code = EFX_QWORD_FIELD(*event, FSF_AZ_DRIVER_EV_SUBCODE); 1196 ev_sub_data = EFX_QWORD_FIELD(*event, FSF_AZ_DRIVER_EV_SUBDATA); 1197 1198 switch (ev_sub_code) { 1199 case FSE_AZ_TX_DESCQ_FLS_DONE_EV: 1200 netif_vdbg(efx, hw, efx->net_dev, "channel %d TXQ %d flushed\n", 1201 channel->channel, ev_sub_data); 1202 efx_farch_handle_tx_flush_done(efx, event); 1203 efx_siena_sriov_tx_flush_done(efx, event); 1204 break; 1205 case FSE_AZ_RX_DESCQ_FLS_DONE_EV: 1206 netif_vdbg(efx, hw, efx->net_dev, "channel %d RXQ %d flushed\n", 1207 channel->channel, ev_sub_data); 1208 efx_farch_handle_rx_flush_done(efx, event); 1209 efx_siena_sriov_rx_flush_done(efx, event); 1210 break; 1211 case FSE_AZ_EVQ_INIT_DONE_EV: 1212 netif_dbg(efx, hw, efx->net_dev, 1213 "channel %d EVQ %d initialised\n", 1214 channel->channel, ev_sub_data); 1215 break; 1216 case FSE_AZ_SRM_UPD_DONE_EV: 1217 netif_vdbg(efx, hw, efx->net_dev, 1218 "channel %d SRAM update done\n", channel->channel); 1219 break; 1220 case FSE_AZ_WAKE_UP_EV: 1221 netif_vdbg(efx, hw, efx->net_dev, 1222 "channel %d RXQ %d wakeup event\n", 1223 channel->channel, ev_sub_data); 1224 break; 1225 case FSE_AZ_TIMER_EV: 1226 netif_vdbg(efx, hw, efx->net_dev, 1227 "channel %d RX queue %d timer expired\n", 1228 channel->channel, ev_sub_data); 1229 break; 1230 case FSE_AA_RX_RECOVER_EV: 1231 netif_err(efx, rx_err, efx->net_dev, 1232 "channel %d seen DRIVER RX_RESET event. " 1233 "Resetting.\n", channel->channel); 1234 atomic_inc(&efx->rx_reset); 1235 efx_schedule_reset(efx, 1236 EFX_WORKAROUND_6555(efx) ? 1237 RESET_TYPE_RX_RECOVERY : 1238 RESET_TYPE_DISABLE); 1239 break; 1240 case FSE_BZ_RX_DSC_ERROR_EV: 1241 if (ev_sub_data < EFX_VI_BASE) { 1242 netif_err(efx, rx_err, efx->net_dev, 1243 "RX DMA Q %d reports descriptor fetch error." 1244 " RX Q %d is disabled.\n", ev_sub_data, 1245 ev_sub_data); 1246 efx_schedule_reset(efx, RESET_TYPE_DMA_ERROR); 1247 } else 1248 efx_siena_sriov_desc_fetch_err(efx, ev_sub_data); 1249 break; 1250 case FSE_BZ_TX_DSC_ERROR_EV: 1251 if (ev_sub_data < EFX_VI_BASE) { 1252 netif_err(efx, tx_err, efx->net_dev, 1253 "TX DMA Q %d reports descriptor fetch error." 1254 " TX Q %d is disabled.\n", ev_sub_data, 1255 ev_sub_data); 1256 efx_schedule_reset(efx, RESET_TYPE_DMA_ERROR); 1257 } else 1258 efx_siena_sriov_desc_fetch_err(efx, ev_sub_data); 1259 break; 1260 default: 1261 netif_vdbg(efx, hw, efx->net_dev, 1262 "channel %d unknown driver event code %d " 1263 "data %04x\n", channel->channel, ev_sub_code, 1264 ev_sub_data); 1265 break; 1266 } 1267} 1268 1269int efx_farch_ev_process(struct efx_channel *channel, int budget) 1270{ 1271 struct efx_nic *efx = channel->efx; 1272 unsigned int read_ptr; 1273 efx_qword_t event, *p_event; 1274 int ev_code; 1275 int tx_packets = 0; 1276 int spent = 0; 1277 1278 if (budget <= 0) 1279 return spent; 1280 1281 read_ptr = channel->eventq_read_ptr; 1282 1283 for (;;) { 1284 p_event = efx_event(channel, read_ptr); 1285 event = *p_event; 1286 1287 if (!efx_event_present(&event)) 1288 /* End of events */ 1289 break; 1290 1291 netif_vdbg(channel->efx, intr, channel->efx->net_dev, 1292 "channel %d event is "EFX_QWORD_FMT"\n", 1293 channel->channel, EFX_QWORD_VAL(event)); 1294 1295 /* Clear this event by marking it all ones */ 1296 EFX_SET_QWORD(*p_event); 1297 1298 ++read_ptr; 1299 1300 ev_code = EFX_QWORD_FIELD(event, FSF_AZ_EV_CODE); 1301 1302 switch (ev_code) { 1303 case FSE_AZ_EV_CODE_RX_EV: 1304 efx_farch_handle_rx_event(channel, &event); 1305 if (++spent == budget) 1306 goto out; 1307 break; 1308 case FSE_AZ_EV_CODE_TX_EV: 1309 tx_packets += efx_farch_handle_tx_event(channel, 1310 &event); 1311 if (tx_packets > efx->txq_entries) { 1312 spent = budget; 1313 goto out; 1314 } 1315 break; 1316 case FSE_AZ_EV_CODE_DRV_GEN_EV: 1317 efx_farch_handle_generated_event(channel, &event); 1318 break; 1319 case FSE_AZ_EV_CODE_DRIVER_EV: 1320 efx_farch_handle_driver_event(channel, &event); 1321 break; 1322 case FSE_CZ_EV_CODE_USER_EV: 1323 efx_siena_sriov_event(channel, &event); 1324 break; 1325 case FSE_CZ_EV_CODE_MCDI_EV: 1326 efx_mcdi_process_event(channel, &event); 1327 break; 1328 case FSE_AZ_EV_CODE_GLOBAL_EV: 1329 if (efx->type->handle_global_event && 1330 efx->type->handle_global_event(channel, &event)) 1331 break; 1332 /* else fall through */ 1333 default: 1334 netif_err(channel->efx, hw, channel->efx->net_dev, 1335 "channel %d unknown event type %d (data " 1336 EFX_QWORD_FMT ")\n", channel->channel, 1337 ev_code, EFX_QWORD_VAL(event)); 1338 } 1339 } 1340 1341out: 1342 channel->eventq_read_ptr = read_ptr; 1343 return spent; 1344} 1345 1346/* Allocate buffer table entries for event queue */ 1347int efx_farch_ev_probe(struct efx_channel *channel) 1348{ 1349 struct efx_nic *efx = channel->efx; 1350 unsigned entries; 1351 1352 entries = channel->eventq_mask + 1; 1353 return efx_alloc_special_buffer(efx, &channel->eventq, 1354 entries * sizeof(efx_qword_t)); 1355} 1356 1357int efx_farch_ev_init(struct efx_channel *channel) 1358{ 1359 efx_oword_t reg; 1360 struct efx_nic *efx = channel->efx; 1361 1362 netif_dbg(efx, hw, efx->net_dev, 1363 "channel %d event queue in special buffers %d-%d\n", 1364 channel->channel, channel->eventq.index, 1365 channel->eventq.index + channel->eventq.entries - 1); 1366 1367 if (efx_nic_rev(efx) >= EFX_REV_SIENA_A0) { 1368 EFX_POPULATE_OWORD_3(reg, 1369 FRF_CZ_TIMER_Q_EN, 1, 1370 FRF_CZ_HOST_NOTIFY_MODE, 0, 1371 FRF_CZ_TIMER_MODE, FFE_CZ_TIMER_MODE_DIS); 1372 efx_writeo_table(efx, ®, FR_BZ_TIMER_TBL, channel->channel); 1373 } 1374 1375 /* Pin event queue buffer */ 1376 efx_init_special_buffer(efx, &channel->eventq); 1377 1378 /* Fill event queue with all ones (i.e. empty events) */ 1379 memset(channel->eventq.buf.addr, 0xff, channel->eventq.buf.len); 1380 1381 /* Push event queue to card */ 1382 EFX_POPULATE_OWORD_3(reg, 1383 FRF_AZ_EVQ_EN, 1, 1384 FRF_AZ_EVQ_SIZE, __ffs(channel->eventq.entries), 1385 FRF_AZ_EVQ_BUF_BASE_ID, channel->eventq.index); 1386 efx_writeo_table(efx, ®, efx->type->evq_ptr_tbl_base, 1387 channel->channel); 1388 1389 return 0; 1390} 1391 1392void efx_farch_ev_fini(struct efx_channel *channel) 1393{ 1394 efx_oword_t reg; 1395 struct efx_nic *efx = channel->efx; 1396 1397 /* Remove event queue from card */ 1398 EFX_ZERO_OWORD(reg); 1399 efx_writeo_table(efx, ®, efx->type->evq_ptr_tbl_base, 1400 channel->channel); 1401 if (efx_nic_rev(efx) >= EFX_REV_SIENA_A0) 1402 efx_writeo_table(efx, ®, FR_BZ_TIMER_TBL, channel->channel); 1403 1404 /* Unpin event queue */ 1405 efx_fini_special_buffer(efx, &channel->eventq); 1406} 1407 1408/* Free buffers backing event queue */ 1409void efx_farch_ev_remove(struct efx_channel *channel) 1410{ 1411 efx_free_special_buffer(channel->efx, &channel->eventq); 1412} 1413 1414 1415void efx_farch_ev_test_generate(struct efx_channel *channel) 1416{ 1417 efx_farch_magic_event(channel, EFX_CHANNEL_MAGIC_TEST(channel)); 1418} 1419 1420void efx_farch_rx_defer_refill(struct efx_rx_queue *rx_queue) 1421{ 1422 efx_farch_magic_event(efx_rx_queue_channel(rx_queue), 1423 EFX_CHANNEL_MAGIC_FILL(rx_queue)); 1424} 1425 1426/************************************************************************** 1427 * 1428 * Hardware interrupts 1429 * The hardware interrupt handler does very little work; all the event 1430 * queue processing is carried out by per-channel tasklets. 1431 * 1432 **************************************************************************/ 1433 1434/* Enable/disable/generate interrupts */ 1435static inline void efx_farch_interrupts(struct efx_nic *efx, 1436 bool enabled, bool force) 1437{ 1438 efx_oword_t int_en_reg_ker; 1439 1440 EFX_POPULATE_OWORD_3(int_en_reg_ker, 1441 FRF_AZ_KER_INT_LEVE_SEL, efx->irq_level, 1442 FRF_AZ_KER_INT_KER, force, 1443 FRF_AZ_DRV_INT_EN_KER, enabled); 1444 efx_writeo(efx, &int_en_reg_ker, FR_AZ_INT_EN_KER); 1445} 1446 1447void efx_farch_irq_enable_master(struct efx_nic *efx) 1448{ 1449 EFX_ZERO_OWORD(*((efx_oword_t *) efx->irq_status.addr)); 1450 wmb(); /* Ensure interrupt vector is clear before interrupts enabled */ 1451 1452 efx_farch_interrupts(efx, true, false); 1453} 1454 1455void efx_farch_irq_disable_master(struct efx_nic *efx) 1456{ 1457 /* Disable interrupts */ 1458 efx_farch_interrupts(efx, false, false); 1459} 1460 1461/* Generate a test interrupt 1462 * Interrupt must already have been enabled, otherwise nasty things 1463 * may happen. 1464 */ 1465void efx_farch_irq_test_generate(struct efx_nic *efx) 1466{ 1467 efx_farch_interrupts(efx, true, true); 1468} 1469 1470/* Process a fatal interrupt 1471 * Disable bus mastering ASAP and schedule a reset 1472 */ 1473irqreturn_t efx_farch_fatal_interrupt(struct efx_nic *efx) 1474{ 1475 struct falcon_nic_data *nic_data = efx->nic_data; 1476 efx_oword_t *int_ker = efx->irq_status.addr; 1477 efx_oword_t fatal_intr; 1478 int error, mem_perr; 1479 1480 efx_reado(efx, &fatal_intr, FR_AZ_FATAL_INTR_KER); 1481 error = EFX_OWORD_FIELD(fatal_intr, FRF_AZ_FATAL_INTR); 1482 1483 netif_err(efx, hw, efx->net_dev, "SYSTEM ERROR "EFX_OWORD_FMT" status " 1484 EFX_OWORD_FMT ": %s\n", EFX_OWORD_VAL(*int_ker), 1485 EFX_OWORD_VAL(fatal_intr), 1486 error ? "disabling bus mastering" : "no recognised error"); 1487 1488 /* If this is a memory parity error dump which blocks are offending */ 1489 mem_perr = (EFX_OWORD_FIELD(fatal_intr, FRF_AZ_MEM_PERR_INT_KER) || 1490 EFX_OWORD_FIELD(fatal_intr, FRF_AZ_SRM_PERR_INT_KER)); 1491 if (mem_perr) { 1492 efx_oword_t reg; 1493 efx_reado(efx, ®, FR_AZ_MEM_STAT); 1494 netif_err(efx, hw, efx->net_dev, 1495 "SYSTEM ERROR: memory parity error "EFX_OWORD_FMT"\n", 1496 EFX_OWORD_VAL(reg)); 1497 } 1498 1499 /* Disable both devices */ 1500 pci_clear_master(efx->pci_dev); 1501 if (efx_nic_is_dual_func(efx)) 1502 pci_clear_master(nic_data->pci_dev2); 1503 efx_farch_irq_disable_master(efx); 1504 1505 /* Count errors and reset or disable the NIC accordingly */ 1506 if (efx->int_error_count == 0 || 1507 time_after(jiffies, efx->int_error_expire)) { 1508 efx->int_error_count = 0; 1509 efx->int_error_expire = 1510 jiffies + EFX_INT_ERROR_EXPIRE * HZ; 1511 } 1512 if (++efx->int_error_count < EFX_MAX_INT_ERRORS) { 1513 netif_err(efx, hw, efx->net_dev, 1514 "SYSTEM ERROR - reset scheduled\n"); 1515 efx_schedule_reset(efx, RESET_TYPE_INT_ERROR); 1516 } else { 1517 netif_err(efx, hw, efx->net_dev, 1518 "SYSTEM ERROR - max number of errors seen." 1519 "NIC will be disabled\n"); 1520 efx_schedule_reset(efx, RESET_TYPE_DISABLE); 1521 } 1522 1523 return IRQ_HANDLED; 1524} 1525 1526/* Handle a legacy interrupt 1527 * Acknowledges the interrupt and schedule event queue processing. 1528 */ 1529irqreturn_t efx_farch_legacy_interrupt(int irq, void *dev_id) 1530{ 1531 struct efx_nic *efx = dev_id; 1532 bool soft_enabled = ACCESS_ONCE(efx->irq_soft_enabled); 1533 efx_oword_t *int_ker = efx->irq_status.addr; 1534 irqreturn_t result = IRQ_NONE; 1535 struct efx_channel *channel; 1536 efx_dword_t reg; 1537 u32 queues; 1538 int syserr; 1539 1540 /* Read the ISR which also ACKs the interrupts */ 1541 efx_readd(efx, ®, FR_BZ_INT_ISR0); 1542 queues = EFX_EXTRACT_DWORD(reg, 0, 31); 1543 1544 /* Legacy interrupts are disabled too late by the EEH kernel 1545 * code. Disable them earlier. 1546 * If an EEH error occurred, the read will have returned all ones. 1547 */ 1548 if (EFX_DWORD_IS_ALL_ONES(reg) && efx_try_recovery(efx) && 1549 !efx->eeh_disabled_legacy_irq) { 1550 disable_irq_nosync(efx->legacy_irq); 1551 efx->eeh_disabled_legacy_irq = true; 1552 } 1553 1554 /* Handle non-event-queue sources */ 1555 if (queues & (1U << efx->irq_level) && soft_enabled) { 1556 syserr = EFX_OWORD_FIELD(*int_ker, FSF_AZ_NET_IVEC_FATAL_INT); 1557 if (unlikely(syserr)) 1558 return efx_farch_fatal_interrupt(efx); 1559 efx->last_irq_cpu = raw_smp_processor_id(); 1560 } 1561 1562 if (queues != 0) { 1563 efx->irq_zero_count = 0; 1564 1565 /* Schedule processing of any interrupting queues */ 1566 if (likely(soft_enabled)) { 1567 efx_for_each_channel(channel, efx) { 1568 if (queues & 1) 1569 efx_schedule_channel_irq(channel); 1570 queues >>= 1; 1571 } 1572 } 1573 result = IRQ_HANDLED; 1574 1575 } else { 1576 efx_qword_t *event; 1577 1578 /* Legacy ISR read can return zero once (SF bug 15783) */ 1579 1580 /* We can't return IRQ_HANDLED more than once on seeing ISR=0 1581 * because this might be a shared interrupt. */ 1582 if (efx->irq_zero_count++ == 0) 1583 result = IRQ_HANDLED; 1584 1585 /* Ensure we schedule or rearm all event queues */ 1586 if (likely(soft_enabled)) { 1587 efx_for_each_channel(channel, efx) { 1588 event = efx_event(channel, 1589 channel->eventq_read_ptr); 1590 if (efx_event_present(event)) 1591 efx_schedule_channel_irq(channel); 1592 else 1593 efx_farch_ev_read_ack(channel); 1594 } 1595 } 1596 } 1597 1598 if (result == IRQ_HANDLED) 1599 netif_vdbg(efx, intr, efx->net_dev, 1600 "IRQ %d on CPU %d status " EFX_DWORD_FMT "\n", 1601 irq, raw_smp_processor_id(), EFX_DWORD_VAL(reg)); 1602 1603 return result; 1604} 1605 1606/* Handle an MSI interrupt 1607 * 1608 * Handle an MSI hardware interrupt. This routine schedules event 1609 * queue processing. No interrupt acknowledgement cycle is necessary. 1610 * Also, we never need to check that the interrupt is for us, since 1611 * MSI interrupts cannot be shared. 1612 */ 1613irqreturn_t efx_farch_msi_interrupt(int irq, void *dev_id) 1614{ 1615 struct efx_msi_context *context = dev_id; 1616 struct efx_nic *efx = context->efx; 1617 efx_oword_t *int_ker = efx->irq_status.addr; 1618 int syserr; 1619 1620 netif_vdbg(efx, intr, efx->net_dev, 1621 "IRQ %d on CPU %d status " EFX_OWORD_FMT "\n", 1622 irq, raw_smp_processor_id(), EFX_OWORD_VAL(*int_ker)); 1623 1624 if (!likely(ACCESS_ONCE(efx->irq_soft_enabled))) 1625 return IRQ_HANDLED; 1626 1627 /* Handle non-event-queue sources */ 1628 if (context->index == efx->irq_level) { 1629 syserr = EFX_OWORD_FIELD(*int_ker, FSF_AZ_NET_IVEC_FATAL_INT); 1630 if (unlikely(syserr)) 1631 return efx_farch_fatal_interrupt(efx); 1632 efx->last_irq_cpu = raw_smp_processor_id(); 1633 } 1634 1635 /* Schedule processing of the channel */ 1636 efx_schedule_channel_irq(efx->channel[context->index]); 1637 1638 return IRQ_HANDLED; 1639} 1640 1641/* Setup RSS indirection table. 1642 * This maps from the hash value of the packet to RXQ 1643 */ 1644void efx_farch_rx_push_indir_table(struct efx_nic *efx) 1645{ 1646 size_t i = 0; 1647 efx_dword_t dword; 1648 1649 BUG_ON(efx_nic_rev(efx) < EFX_REV_FALCON_B0); 1650 1651 BUILD_BUG_ON(ARRAY_SIZE(efx->rx_indir_table) != 1652 FR_BZ_RX_INDIRECTION_TBL_ROWS); 1653 1654 for (i = 0; i < FR_BZ_RX_INDIRECTION_TBL_ROWS; i++) { 1655 EFX_POPULATE_DWORD_1(dword, FRF_BZ_IT_QUEUE, 1656 efx->rx_indir_table[i]); 1657 efx_writed(efx, &dword, 1658 FR_BZ_RX_INDIRECTION_TBL + 1659 FR_BZ_RX_INDIRECTION_TBL_STEP * i); 1660 } 1661} 1662 1663/* Looks at available SRAM resources and works out how many queues we 1664 * can support, and where things like descriptor caches should live. 1665 * 1666 * SRAM is split up as follows: 1667 * 0 buftbl entries for channels 1668 * efx->vf_buftbl_base buftbl entries for SR-IOV 1669 * efx->rx_dc_base RX descriptor caches 1670 * efx->tx_dc_base TX descriptor caches 1671 */ 1672void efx_farch_dimension_resources(struct efx_nic *efx, unsigned sram_lim_qw) 1673{ 1674 unsigned vi_count, buftbl_min; 1675 1676#ifdef CONFIG_SFC_SRIOV 1677 struct siena_nic_data *nic_data = efx->nic_data; 1678#endif 1679 1680 /* Account for the buffer table entries backing the datapath channels 1681 * and the descriptor caches for those channels. 1682 */ 1683 buftbl_min = ((efx->n_rx_channels * EFX_MAX_DMAQ_SIZE + 1684 efx->n_tx_channels * EFX_TXQ_TYPES * EFX_MAX_DMAQ_SIZE + 1685 efx->n_channels * EFX_MAX_EVQ_SIZE) 1686 * sizeof(efx_qword_t) / EFX_BUF_SIZE); 1687 vi_count = max(efx->n_channels, efx->n_tx_channels * EFX_TXQ_TYPES); 1688 1689#ifdef CONFIG_SFC_SRIOV 1690 if (efx->type->sriov_wanted(efx)) { 1691 unsigned vi_dc_entries, buftbl_free, entries_per_vf, vf_limit; 1692 1693 nic_data->vf_buftbl_base = buftbl_min; 1694 1695 vi_dc_entries = RX_DC_ENTRIES + TX_DC_ENTRIES; 1696 vi_count = max(vi_count, EFX_VI_BASE); 1697 buftbl_free = (sram_lim_qw - buftbl_min - 1698 vi_count * vi_dc_entries); 1699 1700 entries_per_vf = ((vi_dc_entries + EFX_VF_BUFTBL_PER_VI) * 1701 efx_vf_size(efx)); 1702 vf_limit = min(buftbl_free / entries_per_vf, 1703 (1024U - EFX_VI_BASE) >> efx->vi_scale); 1704 1705 if (efx->vf_count > vf_limit) { 1706 netif_err(efx, probe, efx->net_dev, 1707 "Reducing VF count from from %d to %d\n", 1708 efx->vf_count, vf_limit); 1709 efx->vf_count = vf_limit; 1710 } 1711 vi_count += efx->vf_count * efx_vf_size(efx); 1712 } 1713#endif 1714 1715 efx->tx_dc_base = sram_lim_qw - vi_count * TX_DC_ENTRIES; 1716 efx->rx_dc_base = efx->tx_dc_base - vi_count * RX_DC_ENTRIES; 1717} 1718 1719u32 efx_farch_fpga_ver(struct efx_nic *efx) 1720{ 1721 efx_oword_t altera_build; 1722 efx_reado(efx, &altera_build, FR_AZ_ALTERA_BUILD); 1723 return EFX_OWORD_FIELD(altera_build, FRF_AZ_ALTERA_BUILD_VER); 1724} 1725 1726void efx_farch_init_common(struct efx_nic *efx) 1727{ 1728 efx_oword_t temp; 1729 1730 /* Set positions of descriptor caches in SRAM. */ 1731 EFX_POPULATE_OWORD_1(temp, FRF_AZ_SRM_TX_DC_BASE_ADR, efx->tx_dc_base); 1732 efx_writeo(efx, &temp, FR_AZ_SRM_TX_DC_CFG); 1733 EFX_POPULATE_OWORD_1(temp, FRF_AZ_SRM_RX_DC_BASE_ADR, efx->rx_dc_base); 1734 efx_writeo(efx, &temp, FR_AZ_SRM_RX_DC_CFG); 1735 1736 /* Set TX descriptor cache size. */ 1737 BUILD_BUG_ON(TX_DC_ENTRIES != (8 << TX_DC_ENTRIES_ORDER)); 1738 EFX_POPULATE_OWORD_1(temp, FRF_AZ_TX_DC_SIZE, TX_DC_ENTRIES_ORDER); 1739 efx_writeo(efx, &temp, FR_AZ_TX_DC_CFG); 1740 1741 /* Set RX descriptor cache size. Set low watermark to size-8, as 1742 * this allows most efficient prefetching. 1743 */ 1744 BUILD_BUG_ON(RX_DC_ENTRIES != (8 << RX_DC_ENTRIES_ORDER)); 1745 EFX_POPULATE_OWORD_1(temp, FRF_AZ_RX_DC_SIZE, RX_DC_ENTRIES_ORDER); 1746 efx_writeo(efx, &temp, FR_AZ_RX_DC_CFG); 1747 EFX_POPULATE_OWORD_1(temp, FRF_AZ_RX_DC_PF_LWM, RX_DC_ENTRIES - 8); 1748 efx_writeo(efx, &temp, FR_AZ_RX_DC_PF_WM); 1749 1750 /* Program INT_KER address */ 1751 EFX_POPULATE_OWORD_2(temp, 1752 FRF_AZ_NORM_INT_VEC_DIS_KER, 1753 EFX_INT_MODE_USE_MSI(efx), 1754 FRF_AZ_INT_ADR_KER, efx->irq_status.dma_addr); 1755 efx_writeo(efx, &temp, FR_AZ_INT_ADR_KER); 1756 1757 if (EFX_WORKAROUND_17213(efx) && !EFX_INT_MODE_USE_MSI(efx)) 1758 /* Use an interrupt level unused by event queues */ 1759 efx->irq_level = 0x1f; 1760 else 1761 /* Use a valid MSI-X vector */ 1762 efx->irq_level = 0; 1763 1764 /* Enable all the genuinely fatal interrupts. (They are still 1765 * masked by the overall interrupt mask, controlled by 1766 * falcon_interrupts()). 1767 * 1768 * Note: All other fatal interrupts are enabled 1769 */ 1770 EFX_POPULATE_OWORD_3(temp, 1771 FRF_AZ_ILL_ADR_INT_KER_EN, 1, 1772 FRF_AZ_RBUF_OWN_INT_KER_EN, 1, 1773 FRF_AZ_TBUF_OWN_INT_KER_EN, 1); 1774 if (efx_nic_rev(efx) >= EFX_REV_SIENA_A0) 1775 EFX_SET_OWORD_FIELD(temp, FRF_CZ_SRAM_PERR_INT_P_KER_EN, 1); 1776 EFX_INVERT_OWORD(temp); 1777 efx_writeo(efx, &temp, FR_AZ_FATAL_INTR_KER); 1778 1779 /* Disable the ugly timer-based TX DMA backoff and allow TX DMA to be 1780 * controlled by the RX FIFO fill level. Set arbitration to one pkt/Q. 1781 */ 1782 efx_reado(efx, &temp, FR_AZ_TX_RESERVED); 1783 EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_RX_SPACER, 0xfe); 1784 EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_RX_SPACER_EN, 1); 1785 EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_ONE_PKT_PER_Q, 1); 1786 EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_PUSH_EN, 1); 1787 EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_DIS_NON_IP_EV, 1); 1788 /* Enable SW_EV to inherit in char driver - assume harmless here */ 1789 EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_SOFT_EVT_EN, 1); 1790 /* Prefetch threshold 2 => fetch when descriptor cache half empty */ 1791 EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_PREF_THRESHOLD, 2); 1792 /* Disable hardware watchdog which can misfire */ 1793 EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff); 1794 /* Squash TX of packets of 16 bytes or less */ 1795 if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) 1796 EFX_SET_OWORD_FIELD(temp, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1); 1797 efx_writeo(efx, &temp, FR_AZ_TX_RESERVED); 1798 1799 if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) { 1800 EFX_POPULATE_OWORD_4(temp, 1801 /* Default values */ 1802 FRF_BZ_TX_PACE_SB_NOT_AF, 0x15, 1803 FRF_BZ_TX_PACE_SB_AF, 0xb, 1804 FRF_BZ_TX_PACE_FB_BASE, 0, 1805 /* Allow large pace values in the 1806 * fast bin. */ 1807 FRF_BZ_TX_PACE_BIN_TH, 1808 FFE_BZ_TX_PACE_RESERVED); 1809 efx_writeo(efx, &temp, FR_BZ_TX_PACE); 1810 } 1811} 1812 1813/************************************************************************** 1814 * 1815 * Filter tables 1816 * 1817 ************************************************************************** 1818 */ 1819 1820/* "Fudge factors" - difference between programmed value and actual depth. 1821 * Due to pipelined implementation we need to program H/W with a value that 1822 * is larger than the hop limit we want. 1823 */ 1824#define EFX_FARCH_FILTER_CTL_SRCH_FUDGE_WILD 3 1825#define EFX_FARCH_FILTER_CTL_SRCH_FUDGE_FULL 1 1826 1827/* Hard maximum search limit. Hardware will time-out beyond 200-something. 1828 * We also need to avoid infinite loops in efx_farch_filter_search() when the 1829 * table is full. 1830 */ 1831#define EFX_FARCH_FILTER_CTL_SRCH_MAX 200 1832 1833/* Don't try very hard to find space for performance hints, as this is 1834 * counter-productive. */ 1835#define EFX_FARCH_FILTER_CTL_SRCH_HINT_MAX 5 1836 1837enum efx_farch_filter_type { 1838 EFX_FARCH_FILTER_TCP_FULL = 0, 1839 EFX_FARCH_FILTER_TCP_WILD, 1840 EFX_FARCH_FILTER_UDP_FULL, 1841 EFX_FARCH_FILTER_UDP_WILD, 1842 EFX_FARCH_FILTER_MAC_FULL = 4, 1843 EFX_FARCH_FILTER_MAC_WILD, 1844 EFX_FARCH_FILTER_UC_DEF = 8, 1845 EFX_FARCH_FILTER_MC_DEF, 1846 EFX_FARCH_FILTER_TYPE_COUNT, /* number of specific types */ 1847}; 1848 1849enum efx_farch_filter_table_id { 1850 EFX_FARCH_FILTER_TABLE_RX_IP = 0, 1851 EFX_FARCH_FILTER_TABLE_RX_MAC, 1852 EFX_FARCH_FILTER_TABLE_RX_DEF, 1853 EFX_FARCH_FILTER_TABLE_TX_MAC, 1854 EFX_FARCH_FILTER_TABLE_COUNT, 1855}; 1856 1857enum efx_farch_filter_index { 1858 EFX_FARCH_FILTER_INDEX_UC_DEF, 1859 EFX_FARCH_FILTER_INDEX_MC_DEF, 1860 EFX_FARCH_FILTER_SIZE_RX_DEF, 1861}; 1862 1863struct efx_farch_filter_spec { 1864 u8 type:4; 1865 u8 priority:4; 1866 u8 flags; 1867 u16 dmaq_id; 1868 u32 data[3]; 1869}; 1870 1871struct efx_farch_filter_table { 1872 enum efx_farch_filter_table_id id; 1873 u32 offset; /* address of table relative to BAR */ 1874 unsigned size; /* number of entries */ 1875 unsigned step; /* step between entries */ 1876 unsigned used; /* number currently used */ 1877 unsigned long *used_bitmap; 1878 struct efx_farch_filter_spec *spec; 1879 unsigned search_limit[EFX_FARCH_FILTER_TYPE_COUNT]; 1880}; 1881 1882struct efx_farch_filter_state { 1883 struct efx_farch_filter_table table[EFX_FARCH_FILTER_TABLE_COUNT]; 1884}; 1885 1886static void 1887efx_farch_filter_table_clear_entry(struct efx_nic *efx, 1888 struct efx_farch_filter_table *table, 1889 unsigned int filter_idx); 1890 1891/* The filter hash function is LFSR polynomial x^16 + x^3 + 1 of a 32-bit 1892 * key derived from the n-tuple. The initial LFSR state is 0xffff. */ 1893static u16 efx_farch_filter_hash(u32 key) 1894{ 1895 u16 tmp; 1896 1897 /* First 16 rounds */ 1898 tmp = 0x1fff ^ key >> 16; 1899 tmp = tmp ^ tmp >> 3 ^ tmp >> 6; 1900 tmp = tmp ^ tmp >> 9; 1901 /* Last 16 rounds */ 1902 tmp = tmp ^ tmp << 13 ^ key; 1903 tmp = tmp ^ tmp >> 3 ^ tmp >> 6; 1904 return tmp ^ tmp >> 9; 1905} 1906 1907/* To allow for hash collisions, filter search continues at these 1908 * increments from the first possible entry selected by the hash. */ 1909static u16 efx_farch_filter_increment(u32 key) 1910{ 1911 return key * 2 - 1; 1912} 1913 1914static enum efx_farch_filter_table_id 1915efx_farch_filter_spec_table_id(const struct efx_farch_filter_spec *spec) 1916{ 1917 BUILD_BUG_ON(EFX_FARCH_FILTER_TABLE_RX_IP != 1918 (EFX_FARCH_FILTER_TCP_FULL >> 2)); 1919 BUILD_BUG_ON(EFX_FARCH_FILTER_TABLE_RX_IP != 1920 (EFX_FARCH_FILTER_TCP_WILD >> 2)); 1921 BUILD_BUG_ON(EFX_FARCH_FILTER_TABLE_RX_IP != 1922 (EFX_FARCH_FILTER_UDP_FULL >> 2)); 1923 BUILD_BUG_ON(EFX_FARCH_FILTER_TABLE_RX_IP != 1924 (EFX_FARCH_FILTER_UDP_WILD >> 2)); 1925 BUILD_BUG_ON(EFX_FARCH_FILTER_TABLE_RX_MAC != 1926 (EFX_FARCH_FILTER_MAC_FULL >> 2)); 1927 BUILD_BUG_ON(EFX_FARCH_FILTER_TABLE_RX_MAC != 1928 (EFX_FARCH_FILTER_MAC_WILD >> 2)); 1929 BUILD_BUG_ON(EFX_FARCH_FILTER_TABLE_TX_MAC != 1930 EFX_FARCH_FILTER_TABLE_RX_MAC + 2); 1931 return (spec->type >> 2) + ((spec->flags & EFX_FILTER_FLAG_TX) ? 2 : 0); 1932} 1933 1934static void efx_farch_filter_push_rx_config(struct efx_nic *efx) 1935{ 1936 struct efx_farch_filter_state *state = efx->filter_state; 1937 struct efx_farch_filter_table *table; 1938 efx_oword_t filter_ctl; 1939 1940 efx_reado(efx, &filter_ctl, FR_BZ_RX_FILTER_CTL); 1941 1942 table = &state->table[EFX_FARCH_FILTER_TABLE_RX_IP]; 1943 EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_TCP_FULL_SRCH_LIMIT, 1944 table->search_limit[EFX_FARCH_FILTER_TCP_FULL] + 1945 EFX_FARCH_FILTER_CTL_SRCH_FUDGE_FULL); 1946 EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_TCP_WILD_SRCH_LIMIT, 1947 table->search_limit[EFX_FARCH_FILTER_TCP_WILD] + 1948 EFX_FARCH_FILTER_CTL_SRCH_FUDGE_WILD); 1949 EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_UDP_FULL_SRCH_LIMIT, 1950 table->search_limit[EFX_FARCH_FILTER_UDP_FULL] + 1951 EFX_FARCH_FILTER_CTL_SRCH_FUDGE_FULL); 1952 EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_UDP_WILD_SRCH_LIMIT, 1953 table->search_limit[EFX_FARCH_FILTER_UDP_WILD] + 1954 EFX_FARCH_FILTER_CTL_SRCH_FUDGE_WILD); 1955 1956 table = &state->table[EFX_FARCH_FILTER_TABLE_RX_MAC]; 1957 if (table->size) { 1958 EFX_SET_OWORD_FIELD( 1959 filter_ctl, FRF_CZ_ETHERNET_FULL_SEARCH_LIMIT, 1960 table->search_limit[EFX_FARCH_FILTER_MAC_FULL] + 1961 EFX_FARCH_FILTER_CTL_SRCH_FUDGE_FULL); 1962 EFX_SET_OWORD_FIELD( 1963 filter_ctl, FRF_CZ_ETHERNET_WILDCARD_SEARCH_LIMIT, 1964 table->search_limit[EFX_FARCH_FILTER_MAC_WILD] + 1965 EFX_FARCH_FILTER_CTL_SRCH_FUDGE_WILD); 1966 } 1967 1968 table = &state->table[EFX_FARCH_FILTER_TABLE_RX_DEF]; 1969 if (table->size) { 1970 EFX_SET_OWORD_FIELD( 1971 filter_ctl, FRF_CZ_UNICAST_NOMATCH_Q_ID, 1972 table->spec[EFX_FARCH_FILTER_INDEX_UC_DEF].dmaq_id); 1973 EFX_SET_OWORD_FIELD( 1974 filter_ctl, FRF_CZ_UNICAST_NOMATCH_RSS_ENABLED, 1975 !!(table->spec[EFX_FARCH_FILTER_INDEX_UC_DEF].flags & 1976 EFX_FILTER_FLAG_RX_RSS)); 1977 EFX_SET_OWORD_FIELD( 1978 filter_ctl, FRF_CZ_MULTICAST_NOMATCH_Q_ID, 1979 table->spec[EFX_FARCH_FILTER_INDEX_MC_DEF].dmaq_id); 1980 EFX_SET_OWORD_FIELD( 1981 filter_ctl, FRF_CZ_MULTICAST_NOMATCH_RSS_ENABLED, 1982 !!(table->spec[EFX_FARCH_FILTER_INDEX_MC_DEF].flags & 1983 EFX_FILTER_FLAG_RX_RSS)); 1984 1985 /* There is a single bit to enable RX scatter for all 1986 * unmatched packets. Only set it if scatter is 1987 * enabled in both filter specs. 1988 */ 1989 EFX_SET_OWORD_FIELD( 1990 filter_ctl, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q, 1991 !!(table->spec[EFX_FARCH_FILTER_INDEX_UC_DEF].flags & 1992 table->spec[EFX_FARCH_FILTER_INDEX_MC_DEF].flags & 1993 EFX_FILTER_FLAG_RX_SCATTER)); 1994 } else if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) { 1995 /* We don't expose 'default' filters because unmatched 1996 * packets always go to the queue number found in the 1997 * RSS table. But we still need to set the RX scatter 1998 * bit here. 1999 */ 2000 EFX_SET_OWORD_FIELD( 2001 filter_ctl, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q, 2002 efx->rx_scatter); 2003 } 2004 2005 efx_writeo(efx, &filter_ctl, FR_BZ_RX_FILTER_CTL); 2006} 2007 2008static void efx_farch_filter_push_tx_limits(struct efx_nic *efx) 2009{ 2010 struct efx_farch_filter_state *state = efx->filter_state; 2011 struct efx_farch_filter_table *table; 2012 efx_oword_t tx_cfg; 2013 2014 efx_reado(efx, &tx_cfg, FR_AZ_TX_CFG); 2015 2016 table = &state->table[EFX_FARCH_FILTER_TABLE_TX_MAC]; 2017 if (table->size) { 2018 EFX_SET_OWORD_FIELD( 2019 tx_cfg, FRF_CZ_TX_ETH_FILTER_FULL_SEARCH_RANGE, 2020 table->search_limit[EFX_FARCH_FILTER_MAC_FULL] + 2021 EFX_FARCH_FILTER_CTL_SRCH_FUDGE_FULL); 2022 EFX_SET_OWORD_FIELD( 2023 tx_cfg, FRF_CZ_TX_ETH_FILTER_WILD_SEARCH_RANGE, 2024 table->search_limit[EFX_FARCH_FILTER_MAC_WILD] + 2025 EFX_FARCH_FILTER_CTL_SRCH_FUDGE_WILD); 2026 } 2027 2028 efx_writeo(efx, &tx_cfg, FR_AZ_TX_CFG); 2029} 2030 2031static int 2032efx_farch_filter_from_gen_spec(struct efx_farch_filter_spec *spec, 2033 const struct efx_filter_spec *gen_spec) 2034{ 2035 bool is_full = false; 2036 2037 if ((gen_spec->flags & EFX_FILTER_FLAG_RX_RSS) && 2038 gen_spec->rss_context != EFX_FILTER_RSS_CONTEXT_DEFAULT) 2039 return -EINVAL; 2040 2041 spec->priority = gen_spec->priority; 2042 spec->flags = gen_spec->flags; 2043 spec->dmaq_id = gen_spec->dmaq_id; 2044 2045 switch (gen_spec->match_flags) { 2046 case (EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO | 2047 EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT | 2048 EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT): 2049 is_full = true; 2050 /* fall through */ 2051 case (EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO | 2052 EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT): { 2053 __be32 rhost, host1, host2; 2054 __be16 rport, port1, port2; 2055 2056 EFX_BUG_ON_PARANOID(!(gen_spec->flags & EFX_FILTER_FLAG_RX)); 2057 2058 if (gen_spec->ether_type != htons(ETH_P_IP)) 2059 return -EPROTONOSUPPORT; 2060 if (gen_spec->loc_port == 0 || 2061 (is_full && gen_spec->rem_port == 0)) 2062 return -EADDRNOTAVAIL; 2063 switch (gen_spec->ip_proto) { 2064 case IPPROTO_TCP: 2065 spec->type = (is_full ? EFX_FARCH_FILTER_TCP_FULL : 2066 EFX_FARCH_FILTER_TCP_WILD); 2067 break; 2068 case IPPROTO_UDP: 2069 spec->type = (is_full ? EFX_FARCH_FILTER_UDP_FULL : 2070 EFX_FARCH_FILTER_UDP_WILD); 2071 break; 2072 default: 2073 return -EPROTONOSUPPORT; 2074 } 2075 2076 /* Filter is constructed in terms of source and destination, 2077 * with the odd wrinkle that the ports are swapped in a UDP 2078 * wildcard filter. We need to convert from local and remote 2079 * (= zero for wildcard) addresses. 2080 */ 2081 rhost = is_full ? gen_spec->rem_host[0] : 0; 2082 rport = is_full ? gen_spec->rem_port : 0; 2083 host1 = rhost; 2084 host2 = gen_spec->loc_host[0]; 2085 if (!is_full && gen_spec->ip_proto == IPPROTO_UDP) { 2086 port1 = gen_spec->loc_port; 2087 port2 = rport; 2088 } else { 2089 port1 = rport; 2090 port2 = gen_spec->loc_port; 2091 } 2092 spec->data[0] = ntohl(host1) << 16 | ntohs(port1); 2093 spec->data[1] = ntohs(port2) << 16 | ntohl(host1) >> 16; 2094 spec->data[2] = ntohl(host2); 2095 2096 break; 2097 } 2098 2099 case EFX_FILTER_MATCH_LOC_MAC | EFX_FILTER_MATCH_OUTER_VID: 2100 is_full = true; 2101 /* fall through */ 2102 case EFX_FILTER_MATCH_LOC_MAC: 2103 spec->type = (is_full ? EFX_FARCH_FILTER_MAC_FULL : 2104 EFX_FARCH_FILTER_MAC_WILD); 2105 spec->data[0] = is_full ? ntohs(gen_spec->outer_vid) : 0; 2106 spec->data[1] = (gen_spec->loc_mac[2] << 24 | 2107 gen_spec->loc_mac[3] << 16 | 2108 gen_spec->loc_mac[4] << 8 | 2109 gen_spec->loc_mac[5]); 2110 spec->data[2] = (gen_spec->loc_mac[0] << 8 | 2111 gen_spec->loc_mac[1]); 2112 break; 2113 2114 case EFX_FILTER_MATCH_LOC_MAC_IG: 2115 spec->type = (is_multicast_ether_addr(gen_spec->loc_mac) ? 2116 EFX_FARCH_FILTER_MC_DEF : 2117 EFX_FARCH_FILTER_UC_DEF); 2118 memset(spec->data, 0, sizeof(spec->data)); /* ensure equality */ 2119 break; 2120 2121 default: 2122 return -EPROTONOSUPPORT; 2123 } 2124 2125 return 0; 2126} 2127 2128static void 2129efx_farch_filter_to_gen_spec(struct efx_filter_spec *gen_spec, 2130 const struct efx_farch_filter_spec *spec) 2131{ 2132 bool is_full = false; 2133 2134 /* *gen_spec should be completely initialised, to be consistent 2135 * with efx_filter_init_{rx,tx}() and in case we want to copy 2136 * it back to userland. 2137 */ 2138 memset(gen_spec, 0, sizeof(*gen_spec)); 2139 2140 gen_spec->priority = spec->priority; 2141 gen_spec->flags = spec->flags; 2142 gen_spec->dmaq_id = spec->dmaq_id; 2143 2144 switch (spec->type) { 2145 case EFX_FARCH_FILTER_TCP_FULL: 2146 case EFX_FARCH_FILTER_UDP_FULL: 2147 is_full = true; 2148 /* fall through */ 2149 case EFX_FARCH_FILTER_TCP_WILD: 2150 case EFX_FARCH_FILTER_UDP_WILD: { 2151 __be32 host1, host2; 2152 __be16 port1, port2; 2153 2154 gen_spec->match_flags = 2155 EFX_FILTER_MATCH_ETHER_TYPE | 2156 EFX_FILTER_MATCH_IP_PROTO | 2157 EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT; 2158 if (is_full) 2159 gen_spec->match_flags |= (EFX_FILTER_MATCH_REM_HOST | 2160 EFX_FILTER_MATCH_REM_PORT); 2161 gen_spec->ether_type = htons(ETH_P_IP); 2162 gen_spec->ip_proto = 2163 (spec->type == EFX_FARCH_FILTER_TCP_FULL || 2164 spec->type == EFX_FARCH_FILTER_TCP_WILD) ? 2165 IPPROTO_TCP : IPPROTO_UDP; 2166 2167 host1 = htonl(spec->data[0] >> 16 | spec->data[1] << 16); 2168 port1 = htons(spec->data[0]); 2169 host2 = htonl(spec->data[2]); 2170 port2 = htons(spec->data[1] >> 16); 2171 if (spec->flags & EFX_FILTER_FLAG_TX) { 2172 gen_spec->loc_host[0] = host1; 2173 gen_spec->rem_host[0] = host2; 2174 } else { 2175 gen_spec->loc_host[0] = host2; 2176 gen_spec->rem_host[0] = host1; 2177 } 2178 if (!!(gen_spec->flags & EFX_FILTER_FLAG_TX) ^ 2179 (!is_full && gen_spec->ip_proto == IPPROTO_UDP)) { 2180 gen_spec->loc_port = port1; 2181 gen_spec->rem_port = port2; 2182 } else { 2183 gen_spec->loc_port = port2; 2184 gen_spec->rem_port = port1; 2185 } 2186 2187 break; 2188 } 2189 2190 case EFX_FARCH_FILTER_MAC_FULL: 2191 is_full = true; 2192 /* fall through */ 2193 case EFX_FARCH_FILTER_MAC_WILD: 2194 gen_spec->match_flags = EFX_FILTER_MATCH_LOC_MAC; 2195 if (is_full) 2196 gen_spec->match_flags |= EFX_FILTER_MATCH_OUTER_VID; 2197 gen_spec->loc_mac[0] = spec->data[2] >> 8; 2198 gen_spec->loc_mac[1] = spec->data[2]; 2199 gen_spec->loc_mac[2] = spec->data[1] >> 24; 2200 gen_spec->loc_mac[3] = spec->data[1] >> 16; 2201 gen_spec->loc_mac[4] = spec->data[1] >> 8; 2202 gen_spec->loc_mac[5] = spec->data[1]; 2203 gen_spec->outer_vid = htons(spec->data[0]); 2204 break; 2205 2206 case EFX_FARCH_FILTER_UC_DEF: 2207 case EFX_FARCH_FILTER_MC_DEF: 2208 gen_spec->match_flags = EFX_FILTER_MATCH_LOC_MAC_IG; 2209 gen_spec->loc_mac[0] = spec->type == EFX_FARCH_FILTER_MC_DEF; 2210 break; 2211 2212 default: 2213 WARN_ON(1); 2214 break; 2215 } 2216} 2217 2218static void 2219efx_farch_filter_init_rx_auto(struct efx_nic *efx, 2220 struct efx_farch_filter_spec *spec) 2221{ 2222 /* If there's only one channel then disable RSS for non VF 2223 * traffic, thereby allowing VFs to use RSS when the PF can't. 2224 */ 2225 spec->priority = EFX_FILTER_PRI_AUTO; 2226 spec->flags = (EFX_FILTER_FLAG_RX | 2227 (efx->n_rx_channels > 1 ? EFX_FILTER_FLAG_RX_RSS : 0) | 2228 (efx->rx_scatter ? EFX_FILTER_FLAG_RX_SCATTER : 0)); 2229 spec->dmaq_id = 0; 2230} 2231 2232/* Build a filter entry and return its n-tuple key. */ 2233static u32 efx_farch_filter_build(efx_oword_t *filter, 2234 struct efx_farch_filter_spec *spec) 2235{ 2236 u32 data3; 2237 2238 switch (efx_farch_filter_spec_table_id(spec)) { 2239 case EFX_FARCH_FILTER_TABLE_RX_IP: { 2240 bool is_udp = (spec->type == EFX_FARCH_FILTER_UDP_FULL || 2241 spec->type == EFX_FARCH_FILTER_UDP_WILD); 2242 EFX_POPULATE_OWORD_7( 2243 *filter, 2244 FRF_BZ_RSS_EN, 2245 !!(spec->flags & EFX_FILTER_FLAG_RX_RSS), 2246 FRF_BZ_SCATTER_EN, 2247 !!(spec->flags & EFX_FILTER_FLAG_RX_SCATTER), 2248 FRF_BZ_TCP_UDP, is_udp, 2249 FRF_BZ_RXQ_ID, spec->dmaq_id, 2250 EFX_DWORD_2, spec->data[2], 2251 EFX_DWORD_1, spec->data[1], 2252 EFX_DWORD_0, spec->data[0]); 2253 data3 = is_udp; 2254 break; 2255 } 2256 2257 case EFX_FARCH_FILTER_TABLE_RX_MAC: { 2258 bool is_wild = spec->type == EFX_FARCH_FILTER_MAC_WILD; 2259 EFX_POPULATE_OWORD_7( 2260 *filter, 2261 FRF_CZ_RMFT_RSS_EN, 2262 !!(spec->flags & EFX_FILTER_FLAG_RX_RSS), 2263 FRF_CZ_RMFT_SCATTER_EN, 2264 !!(spec->flags & EFX_FILTER_FLAG_RX_SCATTER), 2265 FRF_CZ_RMFT_RXQ_ID, spec->dmaq_id, 2266 FRF_CZ_RMFT_WILDCARD_MATCH, is_wild, 2267 FRF_CZ_RMFT_DEST_MAC_HI, spec->data[2], 2268 FRF_CZ_RMFT_DEST_MAC_LO, spec->data[1], 2269 FRF_CZ_RMFT_VLAN_ID, spec->data[0]); 2270 data3 = is_wild; 2271 break; 2272 } 2273 2274 case EFX_FARCH_FILTER_TABLE_TX_MAC: { 2275 bool is_wild = spec->type == EFX_FARCH_FILTER_MAC_WILD; 2276 EFX_POPULATE_OWORD_5(*filter, 2277 FRF_CZ_TMFT_TXQ_ID, spec->dmaq_id, 2278 FRF_CZ_TMFT_WILDCARD_MATCH, is_wild, 2279 FRF_CZ_TMFT_SRC_MAC_HI, spec->data[2], 2280 FRF_CZ_TMFT_SRC_MAC_LO, spec->data[1], 2281 FRF_CZ_TMFT_VLAN_ID, spec->data[0]); 2282 data3 = is_wild | spec->dmaq_id << 1; 2283 break; 2284 } 2285 2286 default: 2287 BUG(); 2288 } 2289 2290 return spec->data[0] ^ spec->data[1] ^ spec->data[2] ^ data3; 2291} 2292 2293static bool efx_farch_filter_equal(const struct efx_farch_filter_spec *left, 2294 const struct efx_farch_filter_spec *right) 2295{ 2296 if (left->type != right->type || 2297 memcmp(left->data, right->data, sizeof(left->data))) 2298 return false; 2299 2300 if (left->flags & EFX_FILTER_FLAG_TX && 2301 left->dmaq_id != right->dmaq_id) 2302 return false; 2303 2304 return true; 2305} 2306 2307/* 2308 * Construct/deconstruct external filter IDs. At least the RX filter 2309 * IDs must be ordered by matching priority, for RX NFC semantics. 2310 * 2311 * Deconstruction needs to be robust against invalid IDs so that 2312 * efx_filter_remove_id_safe() and efx_filter_get_filter_safe() can 2313 * accept user-provided IDs. 2314 */ 2315 2316#define EFX_FARCH_FILTER_MATCH_PRI_COUNT 5 2317 2318static const u8 efx_farch_filter_type_match_pri[EFX_FARCH_FILTER_TYPE_COUNT] = { 2319 [EFX_FARCH_FILTER_TCP_FULL] = 0, 2320 [EFX_FARCH_FILTER_UDP_FULL] = 0, 2321 [EFX_FARCH_FILTER_TCP_WILD] = 1, 2322 [EFX_FARCH_FILTER_UDP_WILD] = 1, 2323 [EFX_FARCH_FILTER_MAC_FULL] = 2, 2324 [EFX_FARCH_FILTER_MAC_WILD] = 3, 2325 [EFX_FARCH_FILTER_UC_DEF] = 4, 2326 [EFX_FARCH_FILTER_MC_DEF] = 4, 2327}; 2328 2329static const enum efx_farch_filter_table_id efx_farch_filter_range_table[] = { 2330 EFX_FARCH_FILTER_TABLE_RX_IP, /* RX match pri 0 */ 2331 EFX_FARCH_FILTER_TABLE_RX_IP, 2332 EFX_FARCH_FILTER_TABLE_RX_MAC, 2333 EFX_FARCH_FILTER_TABLE_RX_MAC, 2334 EFX_FARCH_FILTER_TABLE_RX_DEF, /* RX match pri 4 */ 2335 EFX_FARCH_FILTER_TABLE_TX_MAC, /* TX match pri 0 */ 2336 EFX_FARCH_FILTER_TABLE_TX_MAC, /* TX match pri 1 */ 2337}; 2338 2339#define EFX_FARCH_FILTER_INDEX_WIDTH 13 2340#define EFX_FARCH_FILTER_INDEX_MASK ((1 << EFX_FARCH_FILTER_INDEX_WIDTH) - 1) 2341 2342static inline u32 2343efx_farch_filter_make_id(const struct efx_farch_filter_spec *spec, 2344 unsigned int index) 2345{ 2346 unsigned int range; 2347 2348 range = efx_farch_filter_type_match_pri[spec->type]; 2349 if (!(spec->flags & EFX_FILTER_FLAG_RX)) 2350 range += EFX_FARCH_FILTER_MATCH_PRI_COUNT; 2351 2352 return range << EFX_FARCH_FILTER_INDEX_WIDTH | index; 2353} 2354 2355static inline enum efx_farch_filter_table_id 2356efx_farch_filter_id_table_id(u32 id) 2357{ 2358 unsigned int range = id >> EFX_FARCH_FILTER_INDEX_WIDTH; 2359 2360 if (range < ARRAY_SIZE(efx_farch_filter_range_table)) 2361 return efx_farch_filter_range_table[range]; 2362 else 2363 return EFX_FARCH_FILTER_TABLE_COUNT; /* invalid */ 2364} 2365 2366static inline unsigned int efx_farch_filter_id_index(u32 id) 2367{ 2368 return id & EFX_FARCH_FILTER_INDEX_MASK; 2369} 2370 2371u32 efx_farch_filter_get_rx_id_limit(struct efx_nic *efx) 2372{ 2373 struct efx_farch_filter_state *state = efx->filter_state; 2374 unsigned int range = EFX_FARCH_FILTER_MATCH_PRI_COUNT - 1; 2375 enum efx_farch_filter_table_id table_id; 2376 2377 do { 2378 table_id = efx_farch_filter_range_table[range]; 2379 if (state->table[table_id].size != 0) 2380 return range << EFX_FARCH_FILTER_INDEX_WIDTH | 2381 state->table[table_id].size; 2382 } while (range--); 2383 2384 return 0; 2385} 2386 2387s32 efx_farch_filter_insert(struct efx_nic *efx, 2388 struct efx_filter_spec *gen_spec, 2389 bool replace_equal) 2390{ 2391 struct efx_farch_filter_state *state = efx->filter_state; 2392 struct efx_farch_filter_table *table; 2393 struct efx_farch_filter_spec spec; 2394 efx_oword_t filter; 2395 int rep_index, ins_index; 2396 unsigned int depth = 0; 2397 int rc; 2398 2399 rc = efx_farch_filter_from_gen_spec(&spec, gen_spec); 2400 if (rc) 2401 return rc; 2402 2403 table = &state->table[efx_farch_filter_spec_table_id(&spec)]; 2404 if (table->size == 0) 2405 return -EINVAL; 2406 2407 netif_vdbg(efx, hw, efx->net_dev, 2408 "%s: type %d search_limit=%d", __func__, spec.type, 2409 table->search_limit[spec.type]); 2410 2411 if (table->id == EFX_FARCH_FILTER_TABLE_RX_DEF) { 2412 /* One filter spec per type */ 2413 BUILD_BUG_ON(EFX_FARCH_FILTER_INDEX_UC_DEF != 0); 2414 BUILD_BUG_ON(EFX_FARCH_FILTER_INDEX_MC_DEF != 2415 EFX_FARCH_FILTER_MC_DEF - EFX_FARCH_FILTER_UC_DEF); 2416 rep_index = spec.type - EFX_FARCH_FILTER_UC_DEF; 2417 ins_index = rep_index; 2418 2419 spin_lock_bh(&efx->filter_lock); 2420 } else { 2421 /* Search concurrently for 2422 * (1) a filter to be replaced (rep_index): any filter 2423 * with the same match values, up to the current 2424 * search depth for this type, and 2425 * (2) the insertion point (ins_index): (1) or any 2426 * free slot before it or up to the maximum search 2427 * depth for this priority 2428 * We fail if we cannot find (2). 2429 * 2430 * We can stop once either 2431 * (a) we find (1), in which case we have definitely 2432 * found (2) as well; or 2433 * (b) we have searched exhaustively for (1), and have 2434 * either found (2) or searched exhaustively for it 2435 */ 2436 u32 key = efx_farch_filter_build(&filter, &spec); 2437 unsigned int hash = efx_farch_filter_hash(key); 2438 unsigned int incr = efx_farch_filter_increment(key); 2439 unsigned int max_rep_depth = table->search_limit[spec.type]; 2440 unsigned int max_ins_depth = 2441 spec.priority <= EFX_FILTER_PRI_HINT ? 2442 EFX_FARCH_FILTER_CTL_SRCH_HINT_MAX : 2443 EFX_FARCH_FILTER_CTL_SRCH_MAX; 2444 unsigned int i = hash & (table->size - 1); 2445 2446 ins_index = -1; 2447 depth = 1; 2448 2449 spin_lock_bh(&efx->filter_lock); 2450 2451 for (;;) { 2452 if (!test_bit(i, table->used_bitmap)) { 2453 if (ins_index < 0) 2454 ins_index = i; 2455 } else if (efx_farch_filter_equal(&spec, 2456 &table->spec[i])) { 2457 /* Case (a) */ 2458 if (ins_index < 0) 2459 ins_index = i; 2460 rep_index = i; 2461 break; 2462 } 2463 2464 if (depth >= max_rep_depth && 2465 (ins_index >= 0 || depth >= max_ins_depth)) { 2466 /* Case (b) */ 2467 if (ins_index < 0) { 2468 rc = -EBUSY; 2469 goto out; 2470 } 2471 rep_index = -1; 2472 break; 2473 } 2474 2475 i = (i + incr) & (table->size - 1); 2476 ++depth; 2477 } 2478 } 2479 2480 /* If we found a filter to be replaced, check whether we 2481 * should do so 2482 */ 2483 if (rep_index >= 0) { 2484 struct efx_farch_filter_spec *saved_spec = 2485 &table->spec[rep_index]; 2486 2487 if (spec.priority == saved_spec->priority && !replace_equal) { 2488 rc = -EEXIST; 2489 goto out; 2490 } 2491 if (spec.priority < saved_spec->priority) { 2492 rc = -EPERM; 2493 goto out; 2494 } 2495 if (saved_spec->priority == EFX_FILTER_PRI_AUTO || 2496 saved_spec->flags & EFX_FILTER_FLAG_RX_OVER_AUTO) 2497 spec.flags |= EFX_FILTER_FLAG_RX_OVER_AUTO; 2498 } 2499 2500 /* Insert the filter */ 2501 if (ins_index != rep_index) { 2502 __set_bit(ins_index, table->used_bitmap); 2503 ++table->used; 2504 } 2505 table->spec[ins_index] = spec; 2506 2507 if (table->id == EFX_FARCH_FILTER_TABLE_RX_DEF) { 2508 efx_farch_filter_push_rx_config(efx); 2509 } else { 2510 if (table->search_limit[spec.type] < depth) { 2511 table->search_limit[spec.type] = depth; 2512 if (spec.flags & EFX_FILTER_FLAG_TX) 2513 efx_farch_filter_push_tx_limits(efx); 2514 else 2515 efx_farch_filter_push_rx_config(efx); 2516 } 2517 2518 efx_writeo(efx, &filter, 2519 table->offset + table->step * ins_index); 2520 2521 /* If we were able to replace a filter by inserting 2522 * at a lower depth, clear the replaced filter 2523 */ 2524 if (ins_index != rep_index && rep_index >= 0) 2525 efx_farch_filter_table_clear_entry(efx, table, 2526 rep_index); 2527 } 2528 2529 netif_vdbg(efx, hw, efx->net_dev, 2530 "%s: filter type %d index %d rxq %u set", 2531 __func__, spec.type, ins_index, spec.dmaq_id); 2532 rc = efx_farch_filter_make_id(&spec, ins_index); 2533 2534out: 2535 spin_unlock_bh(&efx->filter_lock); 2536 return rc; 2537} 2538 2539static void 2540efx_farch_filter_table_clear_entry(struct efx_nic *efx, 2541 struct efx_farch_filter_table *table, 2542 unsigned int filter_idx) 2543{ 2544 static efx_oword_t filter; 2545 2546 EFX_WARN_ON_PARANOID(!test_bit(filter_idx, table->used_bitmap)); 2547 BUG_ON(table->offset == 0); /* can't clear MAC default filters */ 2548 2549 __clear_bit(filter_idx, table->used_bitmap); 2550 --table->used; 2551 memset(&table->spec[filter_idx], 0, sizeof(table->spec[0])); 2552 2553 efx_writeo(efx, &filter, table->offset + table->step * filter_idx); 2554 2555 /* If this filter required a greater search depth than 2556 * any other, the search limit for its type can now be 2557 * decreased. However, it is hard to determine that 2558 * unless the table has become completely empty - in 2559 * which case, all its search limits can be set to 0. 2560 */ 2561 if (unlikely(table->used == 0)) { 2562 memset(table->search_limit, 0, sizeof(table->search_limit)); 2563 if (table->id == EFX_FARCH_FILTER_TABLE_TX_MAC) 2564 efx_farch_filter_push_tx_limits(efx); 2565 else 2566 efx_farch_filter_push_rx_config(efx); 2567 } 2568} 2569 2570static int efx_farch_filter_remove(struct efx_nic *efx, 2571 struct efx_farch_filter_table *table, 2572 unsigned int filter_idx, 2573 enum efx_filter_priority priority) 2574{ 2575 struct efx_farch_filter_spec *spec = &table->spec[filter_idx]; 2576 2577 if (!test_bit(filter_idx, table->used_bitmap) || 2578 spec->priority != priority) 2579 return -ENOENT; 2580 2581 if (spec->flags & EFX_FILTER_FLAG_RX_OVER_AUTO) { 2582 efx_farch_filter_init_rx_auto(efx, spec); 2583 efx_farch_filter_push_rx_config(efx); 2584 } else { 2585 efx_farch_filter_table_clear_entry(efx, table, filter_idx); 2586 } 2587 2588 return 0; 2589} 2590 2591int efx_farch_filter_remove_safe(struct efx_nic *efx, 2592 enum efx_filter_priority priority, 2593 u32 filter_id) 2594{ 2595 struct efx_farch_filter_state *state = efx->filter_state; 2596 enum efx_farch_filter_table_id table_id; 2597 struct efx_farch_filter_table *table; 2598 unsigned int filter_idx; 2599 struct efx_farch_filter_spec *spec; 2600 int rc; 2601 2602 table_id = efx_farch_filter_id_table_id(filter_id); 2603 if ((unsigned int)table_id >= EFX_FARCH_FILTER_TABLE_COUNT) 2604 return -ENOENT; 2605 table = &state->table[table_id]; 2606 2607 filter_idx = efx_farch_filter_id_index(filter_id); 2608 if (filter_idx >= table->size) 2609 return -ENOENT; 2610 spec = &table->spec[filter_idx]; 2611 2612 spin_lock_bh(&efx->filter_lock); 2613 rc = efx_farch_filter_remove(efx, table, filter_idx, priority); 2614 spin_unlock_bh(&efx->filter_lock); 2615 2616 return rc; 2617} 2618 2619int efx_farch_filter_get_safe(struct efx_nic *efx, 2620 enum efx_filter_priority priority, 2621 u32 filter_id, struct efx_filter_spec *spec_buf) 2622{ 2623 struct efx_farch_filter_state *state = efx->filter_state; 2624 enum efx_farch_filter_table_id table_id; 2625 struct efx_farch_filter_table *table; 2626 struct efx_farch_filter_spec *spec; 2627 unsigned int filter_idx; 2628 int rc; 2629 2630 table_id = efx_farch_filter_id_table_id(filter_id); 2631 if ((unsigned int)table_id >= EFX_FARCH_FILTER_TABLE_COUNT) 2632 return -ENOENT; 2633 table = &state->table[table_id]; 2634 2635 filter_idx = efx_farch_filter_id_index(filter_id); 2636 if (filter_idx >= table->size) 2637 return -ENOENT; 2638 spec = &table->spec[filter_idx]; 2639 2640 spin_lock_bh(&efx->filter_lock); 2641 2642 if (test_bit(filter_idx, table->used_bitmap) && 2643 spec->priority == priority) { 2644 efx_farch_filter_to_gen_spec(spec_buf, spec); 2645 rc = 0; 2646 } else { 2647 rc = -ENOENT; 2648 } 2649 2650 spin_unlock_bh(&efx->filter_lock); 2651 2652 return rc; 2653} 2654 2655static void 2656efx_farch_filter_table_clear(struct efx_nic *efx, 2657 enum efx_farch_filter_table_id table_id, 2658 enum efx_filter_priority priority) 2659{ 2660 struct efx_farch_filter_state *state = efx->filter_state; 2661 struct efx_farch_filter_table *table = &state->table[table_id]; 2662 unsigned int filter_idx; 2663 2664 spin_lock_bh(&efx->filter_lock); 2665 for (filter_idx = 0; filter_idx < table->size; ++filter_idx) { 2666 if (table->spec[filter_idx].priority != EFX_FILTER_PRI_AUTO) 2667 efx_farch_filter_remove(efx, table, 2668 filter_idx, priority); 2669 } 2670 spin_unlock_bh(&efx->filter_lock); 2671} 2672 2673int efx_farch_filter_clear_rx(struct efx_nic *efx, 2674 enum efx_filter_priority priority) 2675{ 2676 efx_farch_filter_table_clear(efx, EFX_FARCH_FILTER_TABLE_RX_IP, 2677 priority); 2678 efx_farch_filter_table_clear(efx, EFX_FARCH_FILTER_TABLE_RX_MAC, 2679 priority); 2680 efx_farch_filter_table_clear(efx, EFX_FARCH_FILTER_TABLE_RX_DEF, 2681 priority); 2682 return 0; 2683} 2684 2685u32 efx_farch_filter_count_rx_used(struct efx_nic *efx, 2686 enum efx_filter_priority priority) 2687{ 2688 struct efx_farch_filter_state *state = efx->filter_state; 2689 enum efx_farch_filter_table_id table_id; 2690 struct efx_farch_filter_table *table; 2691 unsigned int filter_idx; 2692 u32 count = 0; 2693 2694 spin_lock_bh(&efx->filter_lock); 2695 2696 for (table_id = EFX_FARCH_FILTER_TABLE_RX_IP; 2697 table_id <= EFX_FARCH_FILTER_TABLE_RX_DEF; 2698 table_id++) { 2699 table = &state->table[table_id]; 2700 for (filter_idx = 0; filter_idx < table->size; filter_idx++) { 2701 if (test_bit(filter_idx, table->used_bitmap) && 2702 table->spec[filter_idx].priority == priority) 2703 ++count; 2704 } 2705 } 2706 2707 spin_unlock_bh(&efx->filter_lock); 2708 2709 return count; 2710} 2711 2712s32 efx_farch_filter_get_rx_ids(struct efx_nic *efx, 2713 enum efx_filter_priority priority, 2714 u32 *buf, u32 size) 2715{ 2716 struct efx_farch_filter_state *state = efx->filter_state; 2717 enum efx_farch_filter_table_id table_id; 2718 struct efx_farch_filter_table *table; 2719 unsigned int filter_idx; 2720 s32 count = 0; 2721 2722 spin_lock_bh(&efx->filter_lock); 2723 2724 for (table_id = EFX_FARCH_FILTER_TABLE_RX_IP; 2725 table_id <= EFX_FARCH_FILTER_TABLE_RX_DEF; 2726 table_id++) { 2727 table = &state->table[table_id]; 2728 for (filter_idx = 0; filter_idx < table->size; filter_idx++) { 2729 if (test_bit(filter_idx, table->used_bitmap) && 2730 table->spec[filter_idx].priority == priority) { 2731 if (count == size) { 2732 count = -EMSGSIZE; 2733 goto out; 2734 } 2735 buf[count++] = efx_farch_filter_make_id( 2736 &table->spec[filter_idx], filter_idx); 2737 } 2738 } 2739 } 2740out: 2741 spin_unlock_bh(&efx->filter_lock); 2742 2743 return count; 2744} 2745 2746/* Restore filter stater after reset */ 2747void efx_farch_filter_table_restore(struct efx_nic *efx) 2748{ 2749 struct efx_farch_filter_state *state = efx->filter_state; 2750 enum efx_farch_filter_table_id table_id; 2751 struct efx_farch_filter_table *table; 2752 efx_oword_t filter; 2753 unsigned int filter_idx; 2754 2755 spin_lock_bh(&efx->filter_lock); 2756 2757 for (table_id = 0; table_id < EFX_FARCH_FILTER_TABLE_COUNT; table_id++) { 2758 table = &state->table[table_id]; 2759 2760 /* Check whether this is a regular register table */ 2761 if (table->step == 0) 2762 continue; 2763 2764 for (filter_idx = 0; filter_idx < table->size; filter_idx++) { 2765 if (!test_bit(filter_idx, table->used_bitmap)) 2766 continue; 2767 efx_farch_filter_build(&filter, &table->spec[filter_idx]); 2768 efx_writeo(efx, &filter, 2769 table->offset + table->step * filter_idx); 2770 } 2771 } 2772 2773 efx_farch_filter_push_rx_config(efx); 2774 efx_farch_filter_push_tx_limits(efx); 2775 2776 spin_unlock_bh(&efx->filter_lock); 2777} 2778 2779void efx_farch_filter_table_remove(struct efx_nic *efx) 2780{ 2781 struct efx_farch_filter_state *state = efx->filter_state; 2782 enum efx_farch_filter_table_id table_id; 2783 2784 for (table_id = 0; table_id < EFX_FARCH_FILTER_TABLE_COUNT; table_id++) { 2785 kfree(state->table[table_id].used_bitmap); 2786 vfree(state->table[table_id].spec); 2787 } 2788 kfree(state); 2789} 2790 2791int efx_farch_filter_table_probe(struct efx_nic *efx) 2792{ 2793 struct efx_farch_filter_state *state; 2794 struct efx_farch_filter_table *table; 2795 unsigned table_id; 2796 2797 state = kzalloc(sizeof(struct efx_farch_filter_state), GFP_KERNEL); 2798 if (!state) 2799 return -ENOMEM; 2800 efx->filter_state = state; 2801 2802 if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) { 2803 table = &state->table[EFX_FARCH_FILTER_TABLE_RX_IP]; 2804 table->id = EFX_FARCH_FILTER_TABLE_RX_IP; 2805 table->offset = FR_BZ_RX_FILTER_TBL0; 2806 table->size = FR_BZ_RX_FILTER_TBL0_ROWS; 2807 table->step = FR_BZ_RX_FILTER_TBL0_STEP; 2808 } 2809 2810 if (efx_nic_rev(efx) >= EFX_REV_SIENA_A0) { 2811 table = &state->table[EFX_FARCH_FILTER_TABLE_RX_MAC]; 2812 table->id = EFX_FARCH_FILTER_TABLE_RX_MAC; 2813 table->offset = FR_CZ_RX_MAC_FILTER_TBL0; 2814 table->size = FR_CZ_RX_MAC_FILTER_TBL0_ROWS; 2815 table->step = FR_CZ_RX_MAC_FILTER_TBL0_STEP; 2816 2817 table = &state->table[EFX_FARCH_FILTER_TABLE_RX_DEF]; 2818 table->id = EFX_FARCH_FILTER_TABLE_RX_DEF; 2819 table->size = EFX_FARCH_FILTER_SIZE_RX_DEF; 2820 2821 table = &state->table[EFX_FARCH_FILTER_TABLE_TX_MAC]; 2822 table->id = EFX_FARCH_FILTER_TABLE_TX_MAC; 2823 table->offset = FR_CZ_TX_MAC_FILTER_TBL0; 2824 table->size = FR_CZ_TX_MAC_FILTER_TBL0_ROWS; 2825 table->step = FR_CZ_TX_MAC_FILTER_TBL0_STEP; 2826 } 2827 2828 for (table_id = 0; table_id < EFX_FARCH_FILTER_TABLE_COUNT; table_id++) { 2829 table = &state->table[table_id]; 2830 if (table->size == 0) 2831 continue; 2832 table->used_bitmap = kcalloc(BITS_TO_LONGS(table->size), 2833 sizeof(unsigned long), 2834 GFP_KERNEL); 2835 if (!table->used_bitmap) 2836 goto fail; 2837 table->spec = vzalloc(table->size * sizeof(*table->spec)); 2838 if (!table->spec) 2839 goto fail; 2840 } 2841 2842 table = &state->table[EFX_FARCH_FILTER_TABLE_RX_DEF]; 2843 if (table->size) { 2844 /* RX default filters must always exist */ 2845 struct efx_farch_filter_spec *spec; 2846 unsigned i; 2847 2848 for (i = 0; i < EFX_FARCH_FILTER_SIZE_RX_DEF; i++) { 2849 spec = &table->spec[i]; 2850 spec->type = EFX_FARCH_FILTER_UC_DEF + i; 2851 efx_farch_filter_init_rx_auto(efx, spec); 2852 __set_bit(i, table->used_bitmap); 2853 } 2854 } 2855 2856 efx_farch_filter_push_rx_config(efx); 2857 2858 return 0; 2859 2860fail: 2861 efx_farch_filter_table_remove(efx); 2862 return -ENOMEM; 2863} 2864 2865/* Update scatter enable flags for filters pointing to our own RX queues */ 2866void efx_farch_filter_update_rx_scatter(struct efx_nic *efx) 2867{ 2868 struct efx_farch_filter_state *state = efx->filter_state; 2869 enum efx_farch_filter_table_id table_id; 2870 struct efx_farch_filter_table *table; 2871 efx_oword_t filter; 2872 unsigned int filter_idx; 2873 2874 spin_lock_bh(&efx->filter_lock); 2875 2876 for (table_id = EFX_FARCH_FILTER_TABLE_RX_IP; 2877 table_id <= EFX_FARCH_FILTER_TABLE_RX_DEF; 2878 table_id++) { 2879 table = &state->table[table_id]; 2880 2881 for (filter_idx = 0; filter_idx < table->size; filter_idx++) { 2882 if (!test_bit(filter_idx, table->used_bitmap) || 2883 table->spec[filter_idx].dmaq_id >= 2884 efx->n_rx_channels) 2885 continue; 2886 2887 if (efx->rx_scatter) 2888 table->spec[filter_idx].flags |= 2889 EFX_FILTER_FLAG_RX_SCATTER; 2890 else 2891 table->spec[filter_idx].flags &= 2892 ~EFX_FILTER_FLAG_RX_SCATTER; 2893 2894 if (table_id == EFX_FARCH_FILTER_TABLE_RX_DEF) 2895 /* Pushed by efx_farch_filter_push_rx_config() */ 2896 continue; 2897 2898 efx_farch_filter_build(&filter, &table->spec[filter_idx]); 2899 efx_writeo(efx, &filter, 2900 table->offset + table->step * filter_idx); 2901 } 2902 } 2903 2904 efx_farch_filter_push_rx_config(efx); 2905 2906 spin_unlock_bh(&efx->filter_lock); 2907} 2908 2909#ifdef CONFIG_RFS_ACCEL 2910 2911s32 efx_farch_filter_rfs_insert(struct efx_nic *efx, 2912 struct efx_filter_spec *gen_spec) 2913{ 2914 return efx_farch_filter_insert(efx, gen_spec, true); 2915} 2916 2917bool efx_farch_filter_rfs_expire_one(struct efx_nic *efx, u32 flow_id, 2918 unsigned int index) 2919{ 2920 struct efx_farch_filter_state *state = efx->filter_state; 2921 struct efx_farch_filter_table *table = 2922 &state->table[EFX_FARCH_FILTER_TABLE_RX_IP]; 2923 2924 if (test_bit(index, table->used_bitmap) && 2925 table->spec[index].priority == EFX_FILTER_PRI_HINT && 2926 rps_may_expire_flow(efx->net_dev, table->spec[index].dmaq_id, 2927 flow_id, index)) { 2928 efx_farch_filter_table_clear_entry(efx, table, index); 2929 return true; 2930 } 2931 2932 return false; 2933} 2934 2935#endif /* CONFIG_RFS_ACCEL */ 2936 2937void efx_farch_filter_sync_rx_mode(struct efx_nic *efx) 2938{ 2939 struct net_device *net_dev = efx->net_dev; 2940 struct netdev_hw_addr *ha; 2941 union efx_multicast_hash *mc_hash = &efx->multicast_hash; 2942 u32 crc; 2943 int bit; 2944 2945 if (!efx_dev_registered(efx)) 2946 return; 2947 2948 netif_addr_lock_bh(net_dev); 2949 2950 efx->unicast_filter = !(net_dev->flags & IFF_PROMISC); 2951 2952 /* Build multicast hash table */ 2953 if (net_dev->flags & (IFF_PROMISC | IFF_ALLMULTI)) { 2954 memset(mc_hash, 0xff, sizeof(*mc_hash)); 2955 } else { 2956 memset(mc_hash, 0x00, sizeof(*mc_hash)); 2957 netdev_for_each_mc_addr(ha, net_dev) { 2958 crc = ether_crc_le(ETH_ALEN, ha->addr); 2959 bit = crc & (EFX_MCAST_HASH_ENTRIES - 1); 2960 __set_bit_le(bit, mc_hash); 2961 } 2962 2963 /* Broadcast packets go through the multicast hash filter. 2964 * ether_crc_le() of the broadcast address is 0xbe2612ff 2965 * so we always add bit 0xff to the mask. 2966 */ 2967 __set_bit_le(0xff, mc_hash); 2968 } 2969 2970 netif_addr_unlock_bh(net_dev); 2971} 2972