1/* 2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33#include <asm-generic/kmap_types.h> 34#include <linux/kernel.h> 35#include <linux/module.h> 36#include <linux/mlx5/driver.h> 37#include <linux/mlx5/cmd.h> 38#include "mlx5_core.h" 39 40enum { 41 MLX5_PAGES_CANT_GIVE = 0, 42 MLX5_PAGES_GIVE = 1, 43 MLX5_PAGES_TAKE = 2 44}; 45 46enum { 47 MLX5_BOOT_PAGES = 1, 48 MLX5_INIT_PAGES = 2, 49 MLX5_POST_INIT_PAGES = 3 50}; 51 52struct mlx5_pages_req { 53 struct mlx5_core_dev *dev; 54 u16 func_id; 55 s32 npages; 56 struct work_struct work; 57}; 58 59struct fw_page { 60 struct rb_node rb_node; 61 u64 addr; 62 struct page *page; 63 u16 func_id; 64 unsigned long bitmask; 65 struct list_head list; 66 unsigned free_count; 67}; 68 69struct mlx5_query_pages_inbox { 70 struct mlx5_inbox_hdr hdr; 71 u8 rsvd[8]; 72}; 73 74struct mlx5_query_pages_outbox { 75 struct mlx5_outbox_hdr hdr; 76 __be16 rsvd; 77 __be16 func_id; 78 __be32 num_pages; 79}; 80 81struct mlx5_manage_pages_inbox { 82 struct mlx5_inbox_hdr hdr; 83 __be16 rsvd; 84 __be16 func_id; 85 __be32 num_entries; 86 __be64 pas[0]; 87}; 88 89struct mlx5_manage_pages_outbox { 90 struct mlx5_outbox_hdr hdr; 91 __be32 num_entries; 92 u8 rsvd[4]; 93 __be64 pas[0]; 94}; 95 96enum { 97 MAX_RECLAIM_TIME_MSECS = 5000, 98}; 99 100enum { 101 MLX5_MAX_RECLAIM_TIME_MILI = 5000, 102 MLX5_NUM_4K_IN_PAGE = PAGE_SIZE / MLX5_ADAPTER_PAGE_SIZE, 103}; 104 105static int insert_page(struct mlx5_core_dev *dev, u64 addr, struct page *page, u16 func_id) 106{ 107 struct rb_root *root = &dev->priv.page_root; 108 struct rb_node **new = &root->rb_node; 109 struct rb_node *parent = NULL; 110 struct fw_page *nfp; 111 struct fw_page *tfp; 112 int i; 113 114 while (*new) { 115 parent = *new; 116 tfp = rb_entry(parent, struct fw_page, rb_node); 117 if (tfp->addr < addr) 118 new = &parent->rb_left; 119 else if (tfp->addr > addr) 120 new = &parent->rb_right; 121 else 122 return -EEXIST; 123 } 124 125 nfp = kzalloc(sizeof(*nfp), GFP_KERNEL); 126 if (!nfp) 127 return -ENOMEM; 128 129 nfp->addr = addr; 130 nfp->page = page; 131 nfp->func_id = func_id; 132 nfp->free_count = MLX5_NUM_4K_IN_PAGE; 133 for (i = 0; i < MLX5_NUM_4K_IN_PAGE; i++) 134 set_bit(i, &nfp->bitmask); 135 136 rb_link_node(&nfp->rb_node, parent, new); 137 rb_insert_color(&nfp->rb_node, root); 138 list_add(&nfp->list, &dev->priv.free_list); 139 140 return 0; 141} 142 143static struct fw_page *find_fw_page(struct mlx5_core_dev *dev, u64 addr) 144{ 145 struct rb_root *root = &dev->priv.page_root; 146 struct rb_node *tmp = root->rb_node; 147 struct fw_page *result = NULL; 148 struct fw_page *tfp; 149 150 while (tmp) { 151 tfp = rb_entry(tmp, struct fw_page, rb_node); 152 if (tfp->addr < addr) { 153 tmp = tmp->rb_left; 154 } else if (tfp->addr > addr) { 155 tmp = tmp->rb_right; 156 } else { 157 result = tfp; 158 break; 159 } 160 } 161 162 return result; 163} 164 165static int mlx5_cmd_query_pages(struct mlx5_core_dev *dev, u16 *func_id, 166 s32 *npages, int boot) 167{ 168 struct mlx5_query_pages_inbox in; 169 struct mlx5_query_pages_outbox out; 170 int err; 171 172 memset(&in, 0, sizeof(in)); 173 memset(&out, 0, sizeof(out)); 174 in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_QUERY_PAGES); 175 in.hdr.opmod = boot ? cpu_to_be16(MLX5_BOOT_PAGES) : cpu_to_be16(MLX5_INIT_PAGES); 176 177 err = mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out)); 178 if (err) 179 return err; 180 181 if (out.hdr.status) 182 return mlx5_cmd_status_to_err(&out.hdr); 183 184 *npages = be32_to_cpu(out.num_pages); 185 *func_id = be16_to_cpu(out.func_id); 186 187 return err; 188} 189 190static int alloc_4k(struct mlx5_core_dev *dev, u64 *addr) 191{ 192 struct fw_page *fp; 193 unsigned n; 194 195 if (list_empty(&dev->priv.free_list)) 196 return -ENOMEM; 197 198 fp = list_entry(dev->priv.free_list.next, struct fw_page, list); 199 n = find_first_bit(&fp->bitmask, 8 * sizeof(fp->bitmask)); 200 if (n >= MLX5_NUM_4K_IN_PAGE) { 201 mlx5_core_warn(dev, "alloc 4k bug\n"); 202 return -ENOENT; 203 } 204 clear_bit(n, &fp->bitmask); 205 fp->free_count--; 206 if (!fp->free_count) 207 list_del(&fp->list); 208 209 *addr = fp->addr + n * MLX5_ADAPTER_PAGE_SIZE; 210 211 return 0; 212} 213 214#define MLX5_U64_4K_PAGE_MASK ((~(u64)0U) << PAGE_SHIFT) 215 216static void free_4k(struct mlx5_core_dev *dev, u64 addr) 217{ 218 struct fw_page *fwp; 219 int n; 220 221 fwp = find_fw_page(dev, addr & MLX5_U64_4K_PAGE_MASK); 222 if (!fwp) { 223 mlx5_core_warn(dev, "page not found\n"); 224 return; 225 } 226 227 n = (addr & ~MLX5_U64_4K_PAGE_MASK) >> MLX5_ADAPTER_PAGE_SHIFT; 228 fwp->free_count++; 229 set_bit(n, &fwp->bitmask); 230 if (fwp->free_count == MLX5_NUM_4K_IN_PAGE) { 231 rb_erase(&fwp->rb_node, &dev->priv.page_root); 232 if (fwp->free_count != 1) 233 list_del(&fwp->list); 234 dma_unmap_page(&dev->pdev->dev, addr & MLX5_U64_4K_PAGE_MASK, 235 PAGE_SIZE, DMA_BIDIRECTIONAL); 236 __free_page(fwp->page); 237 kfree(fwp); 238 } else if (fwp->free_count == 1) { 239 list_add(&fwp->list, &dev->priv.free_list); 240 } 241} 242 243static int alloc_system_page(struct mlx5_core_dev *dev, u16 func_id) 244{ 245 struct page *page; 246 u64 addr; 247 int err; 248 int nid = dev_to_node(&dev->pdev->dev); 249 250 page = alloc_pages_node(nid, GFP_HIGHUSER, 0); 251 if (!page) { 252 mlx5_core_warn(dev, "failed to allocate page\n"); 253 return -ENOMEM; 254 } 255 addr = dma_map_page(&dev->pdev->dev, page, 0, 256 PAGE_SIZE, DMA_BIDIRECTIONAL); 257 if (dma_mapping_error(&dev->pdev->dev, addr)) { 258 mlx5_core_warn(dev, "failed dma mapping page\n"); 259 err = -ENOMEM; 260 goto out_alloc; 261 } 262 err = insert_page(dev, addr, page, func_id); 263 if (err) { 264 mlx5_core_err(dev, "failed to track allocated page\n"); 265 goto out_mapping; 266 } 267 268 return 0; 269 270out_mapping: 271 dma_unmap_page(&dev->pdev->dev, addr, PAGE_SIZE, DMA_BIDIRECTIONAL); 272 273out_alloc: 274 __free_page(page); 275 276 return err; 277} 278static int give_pages(struct mlx5_core_dev *dev, u16 func_id, int npages, 279 int notify_fail) 280{ 281 struct mlx5_manage_pages_inbox *in; 282 struct mlx5_manage_pages_outbox out; 283 struct mlx5_manage_pages_inbox *nin; 284 int inlen; 285 u64 addr; 286 int err; 287 int i; 288 289 inlen = sizeof(*in) + npages * sizeof(in->pas[0]); 290 in = mlx5_vzalloc(inlen); 291 if (!in) { 292 mlx5_core_warn(dev, "vzalloc failed %d\n", inlen); 293 return -ENOMEM; 294 } 295 memset(&out, 0, sizeof(out)); 296 297 for (i = 0; i < npages; i++) { 298retry: 299 err = alloc_4k(dev, &addr); 300 if (err) { 301 if (err == -ENOMEM) 302 err = alloc_system_page(dev, func_id); 303 if (err) 304 goto out_4k; 305 306 goto retry; 307 } 308 in->pas[i] = cpu_to_be64(addr); 309 } 310 311 in->hdr.opcode = cpu_to_be16(MLX5_CMD_OP_MANAGE_PAGES); 312 in->hdr.opmod = cpu_to_be16(MLX5_PAGES_GIVE); 313 in->func_id = cpu_to_be16(func_id); 314 in->num_entries = cpu_to_be32(npages); 315 err = mlx5_cmd_exec(dev, in, inlen, &out, sizeof(out)); 316 if (err) { 317 mlx5_core_warn(dev, "func_id 0x%x, npages %d, err %d\n", 318 func_id, npages, err); 319 goto out_alloc; 320 } 321 dev->priv.fw_pages += npages; 322 323 if (out.hdr.status) { 324 err = mlx5_cmd_status_to_err(&out.hdr); 325 if (err) { 326 mlx5_core_warn(dev, "func_id 0x%x, npages %d, status %d\n", 327 func_id, npages, out.hdr.status); 328 goto out_alloc; 329 } 330 } 331 332 mlx5_core_dbg(dev, "err %d\n", err); 333 334 goto out_free; 335 336out_alloc: 337 if (notify_fail) { 338 nin = kzalloc(sizeof(*nin), GFP_KERNEL); 339 if (!nin) { 340 mlx5_core_warn(dev, "allocation failed\n"); 341 goto out_4k; 342 } 343 memset(&out, 0, sizeof(out)); 344 nin->hdr.opcode = cpu_to_be16(MLX5_CMD_OP_MANAGE_PAGES); 345 nin->hdr.opmod = cpu_to_be16(MLX5_PAGES_CANT_GIVE); 346 if (mlx5_cmd_exec(dev, nin, sizeof(*nin), &out, sizeof(out))) 347 mlx5_core_warn(dev, "page notify failed\n"); 348 kfree(nin); 349 } 350 351out_4k: 352 for (i--; i >= 0; i--) 353 free_4k(dev, be64_to_cpu(in->pas[i])); 354out_free: 355 kvfree(in); 356 return err; 357} 358 359static int reclaim_pages(struct mlx5_core_dev *dev, u32 func_id, int npages, 360 int *nclaimed) 361{ 362 struct mlx5_manage_pages_inbox in; 363 struct mlx5_manage_pages_outbox *out; 364 int num_claimed; 365 int outlen; 366 u64 addr; 367 int err; 368 int i; 369 370 if (nclaimed) 371 *nclaimed = 0; 372 373 memset(&in, 0, sizeof(in)); 374 outlen = sizeof(*out) + npages * sizeof(out->pas[0]); 375 out = mlx5_vzalloc(outlen); 376 if (!out) 377 return -ENOMEM; 378 379 in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_MANAGE_PAGES); 380 in.hdr.opmod = cpu_to_be16(MLX5_PAGES_TAKE); 381 in.func_id = cpu_to_be16(func_id); 382 in.num_entries = cpu_to_be32(npages); 383 mlx5_core_dbg(dev, "npages %d, outlen %d\n", npages, outlen); 384 err = mlx5_cmd_exec(dev, &in, sizeof(in), out, outlen); 385 if (err) { 386 mlx5_core_err(dev, "failed reclaiming pages\n"); 387 goto out_free; 388 } 389 dev->priv.fw_pages -= npages; 390 391 if (out->hdr.status) { 392 err = mlx5_cmd_status_to_err(&out->hdr); 393 goto out_free; 394 } 395 396 num_claimed = be32_to_cpu(out->num_entries); 397 if (nclaimed) 398 *nclaimed = num_claimed; 399 400 for (i = 0; i < num_claimed; i++) { 401 addr = be64_to_cpu(out->pas[i]); 402 free_4k(dev, addr); 403 } 404 405out_free: 406 kvfree(out); 407 return err; 408} 409 410static void pages_work_handler(struct work_struct *work) 411{ 412 struct mlx5_pages_req *req = container_of(work, struct mlx5_pages_req, work); 413 struct mlx5_core_dev *dev = req->dev; 414 int err = 0; 415 416 if (req->npages < 0) 417 err = reclaim_pages(dev, req->func_id, -1 * req->npages, NULL); 418 else if (req->npages > 0) 419 err = give_pages(dev, req->func_id, req->npages, 1); 420 421 if (err) 422 mlx5_core_warn(dev, "%s fail %d\n", 423 req->npages < 0 ? "reclaim" : "give", err); 424 425 kfree(req); 426} 427 428void mlx5_core_req_pages_handler(struct mlx5_core_dev *dev, u16 func_id, 429 s32 npages) 430{ 431 struct mlx5_pages_req *req; 432 433 req = kzalloc(sizeof(*req), GFP_ATOMIC); 434 if (!req) { 435 mlx5_core_warn(dev, "failed to allocate pages request\n"); 436 return; 437 } 438 439 req->dev = dev; 440 req->func_id = func_id; 441 req->npages = npages; 442 INIT_WORK(&req->work, pages_work_handler); 443 queue_work(dev->priv.pg_wq, &req->work); 444} 445 446int mlx5_satisfy_startup_pages(struct mlx5_core_dev *dev, int boot) 447{ 448 u16 uninitialized_var(func_id); 449 s32 uninitialized_var(npages); 450 int err; 451 452 err = mlx5_cmd_query_pages(dev, &func_id, &npages, boot); 453 if (err) 454 return err; 455 456 mlx5_core_dbg(dev, "requested %d %s pages for func_id 0x%x\n", 457 npages, boot ? "boot" : "init", func_id); 458 459 return give_pages(dev, func_id, npages, 0); 460} 461 462enum { 463 MLX5_BLKS_FOR_RECLAIM_PAGES = 12 464}; 465 466static int optimal_reclaimed_pages(void) 467{ 468 struct mlx5_cmd_prot_block *block; 469 struct mlx5_cmd_layout *lay; 470 int ret; 471 472 ret = (sizeof(lay->out) + MLX5_BLKS_FOR_RECLAIM_PAGES * sizeof(block->data) - 473 sizeof(struct mlx5_manage_pages_outbox)) / 474 FIELD_SIZEOF(struct mlx5_manage_pages_outbox, pas[0]); 475 476 return ret; 477} 478 479int mlx5_reclaim_startup_pages(struct mlx5_core_dev *dev) 480{ 481 unsigned long end = jiffies + msecs_to_jiffies(MAX_RECLAIM_TIME_MSECS); 482 struct fw_page *fwp; 483 struct rb_node *p; 484 int nclaimed = 0; 485 int err; 486 487 do { 488 p = rb_first(&dev->priv.page_root); 489 if (p) { 490 fwp = rb_entry(p, struct fw_page, rb_node); 491 err = reclaim_pages(dev, fwp->func_id, 492 optimal_reclaimed_pages(), 493 &nclaimed); 494 if (err) { 495 mlx5_core_warn(dev, "failed reclaiming pages (%d)\n", 496 err); 497 return err; 498 } 499 if (nclaimed) 500 end = jiffies + msecs_to_jiffies(MAX_RECLAIM_TIME_MSECS); 501 } 502 if (time_after(jiffies, end)) { 503 mlx5_core_warn(dev, "FW did not return all pages. giving up...\n"); 504 break; 505 } 506 } while (p); 507 508 return 0; 509} 510 511void mlx5_pagealloc_init(struct mlx5_core_dev *dev) 512{ 513 dev->priv.page_root = RB_ROOT; 514 INIT_LIST_HEAD(&dev->priv.free_list); 515} 516 517void mlx5_pagealloc_cleanup(struct mlx5_core_dev *dev) 518{ 519 /* nothing */ 520} 521 522int mlx5_pagealloc_start(struct mlx5_core_dev *dev) 523{ 524 dev->priv.pg_wq = create_singlethread_workqueue("mlx5_page_allocator"); 525 if (!dev->priv.pg_wq) 526 return -ENOMEM; 527 528 return 0; 529} 530 531void mlx5_pagealloc_stop(struct mlx5_core_dev *dev) 532{ 533 destroy_workqueue(dev->priv.pg_wq); 534} 535