1/* 2 * Copyright 2012 Tilera Corporation. All Rights Reserved. 3 * 4 * This program is free software; you can redistribute it and/or 5 * modify it under the terms of the GNU General Public License 6 * as published by the Free Software Foundation, version 2. 7 * 8 * This program is distributed in the hope that it will be useful, but 9 * WITHOUT ANY WARRANTY; without even the implied warranty of 10 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or 11 * NON INFRINGEMENT. See the GNU General Public License for 12 * more details. 13 */ 14 15/* 16 * Implementation of mpipe gxio calls. 17 */ 18 19#include <linux/errno.h> 20#include <linux/io.h> 21#include <linux/module.h> 22 23#include <gxio/iorpc_globals.h> 24#include <gxio/iorpc_mpipe.h> 25#include <gxio/iorpc_mpipe_info.h> 26#include <gxio/kiorpc.h> 27#include <gxio/mpipe.h> 28 29/* HACK: Avoid pointless "shadow" warnings. */ 30#define link link_shadow 31 32/** 33 * strscpy - Copy a C-string into a sized buffer, but only if it fits 34 * @dest: Where to copy the string to 35 * @src: Where to copy the string from 36 * @size: size of destination buffer 37 * 38 * Use this routine to avoid copying too-long strings. 39 * The routine returns the total number of bytes copied 40 * (including the trailing NUL) or zero if the buffer wasn't 41 * big enough. To ensure that programmers pay attention 42 * to the return code, the destination has a single NUL 43 * written at the front (if size is non-zero) when the 44 * buffer is not big enough. 45 */ 46static size_t strscpy(char *dest, const char *src, size_t size) 47{ 48 size_t len = strnlen(src, size) + 1; 49 if (len > size) { 50 if (size) 51 dest[0] = '\0'; 52 return 0; 53 } 54 memcpy(dest, src, len); 55 return len; 56} 57 58int gxio_mpipe_init(gxio_mpipe_context_t *context, unsigned int mpipe_index) 59{ 60 char file[32]; 61 62 int fd; 63 int i; 64 65 if (mpipe_index >= GXIO_MPIPE_INSTANCE_MAX) 66 return -EINVAL; 67 68 snprintf(file, sizeof(file), "mpipe/%d/iorpc", mpipe_index); 69 fd = hv_dev_open((HV_VirtAddr) file, 0); 70 71 context->fd = fd; 72 73 if (fd < 0) { 74 if (fd >= GXIO_ERR_MIN && fd <= GXIO_ERR_MAX) 75 return fd; 76 else 77 return -ENODEV; 78 } 79 80 /* Map in the MMIO space. */ 81 context->mmio_cfg_base = (void __force *) 82 iorpc_ioremap(fd, HV_MPIPE_CONFIG_MMIO_OFFSET, 83 HV_MPIPE_CONFIG_MMIO_SIZE); 84 if (context->mmio_cfg_base == NULL) 85 goto cfg_failed; 86 87 context->mmio_fast_base = (void __force *) 88 iorpc_ioremap(fd, HV_MPIPE_FAST_MMIO_OFFSET, 89 HV_MPIPE_FAST_MMIO_SIZE); 90 if (context->mmio_fast_base == NULL) 91 goto fast_failed; 92 93 /* Initialize the stacks. */ 94 for (i = 0; i < 8; i++) 95 context->__stacks.stacks[i] = 255; 96 97 context->instance = mpipe_index; 98 99 return 0; 100 101 fast_failed: 102 iounmap((void __force __iomem *)(context->mmio_cfg_base)); 103 cfg_failed: 104 hv_dev_close(context->fd); 105 context->fd = -1; 106 return -ENODEV; 107} 108 109EXPORT_SYMBOL_GPL(gxio_mpipe_init); 110 111int gxio_mpipe_destroy(gxio_mpipe_context_t *context) 112{ 113 iounmap((void __force __iomem *)(context->mmio_cfg_base)); 114 iounmap((void __force __iomem *)(context->mmio_fast_base)); 115 return hv_dev_close(context->fd); 116} 117 118EXPORT_SYMBOL_GPL(gxio_mpipe_destroy); 119 120static int16_t gxio_mpipe_buffer_sizes[8] = 121 { 128, 256, 512, 1024, 1664, 4096, 10368, 16384 }; 122 123gxio_mpipe_buffer_size_enum_t gxio_mpipe_buffer_size_to_buffer_size_enum(size_t 124 size) 125{ 126 int i; 127 for (i = 0; i < 7; i++) 128 if (size <= gxio_mpipe_buffer_sizes[i]) 129 break; 130 return i; 131} 132 133EXPORT_SYMBOL_GPL(gxio_mpipe_buffer_size_to_buffer_size_enum); 134 135size_t gxio_mpipe_buffer_size_enum_to_buffer_size(gxio_mpipe_buffer_size_enum_t 136 buffer_size_enum) 137{ 138 if (buffer_size_enum > 7) 139 buffer_size_enum = 7; 140 141 return gxio_mpipe_buffer_sizes[buffer_size_enum]; 142} 143 144EXPORT_SYMBOL_GPL(gxio_mpipe_buffer_size_enum_to_buffer_size); 145 146size_t gxio_mpipe_calc_buffer_stack_bytes(unsigned long buffers) 147{ 148 const int BUFFERS_PER_LINE = 12; 149 150 /* Count the number of cachlines. */ 151 unsigned long lines = 152 (buffers + BUFFERS_PER_LINE - 1) / BUFFERS_PER_LINE; 153 154 /* Convert to bytes. */ 155 return lines * CHIP_L2_LINE_SIZE(); 156} 157 158EXPORT_SYMBOL_GPL(gxio_mpipe_calc_buffer_stack_bytes); 159 160int gxio_mpipe_init_buffer_stack(gxio_mpipe_context_t *context, 161 unsigned int stack, 162 gxio_mpipe_buffer_size_enum_t 163 buffer_size_enum, void *mem, size_t mem_size, 164 unsigned int mem_flags) 165{ 166 int result; 167 168 memset(mem, 0, mem_size); 169 170 result = gxio_mpipe_init_buffer_stack_aux(context, mem, mem_size, 171 mem_flags, stack, 172 buffer_size_enum); 173 if (result < 0) 174 return result; 175 176 /* Save the stack. */ 177 context->__stacks.stacks[buffer_size_enum] = stack; 178 179 return 0; 180} 181 182EXPORT_SYMBOL_GPL(gxio_mpipe_init_buffer_stack); 183 184int gxio_mpipe_init_notif_ring(gxio_mpipe_context_t *context, 185 unsigned int ring, 186 void *mem, size_t mem_size, 187 unsigned int mem_flags) 188{ 189 return gxio_mpipe_init_notif_ring_aux(context, mem, mem_size, 190 mem_flags, ring); 191} 192 193EXPORT_SYMBOL_GPL(gxio_mpipe_init_notif_ring); 194 195int gxio_mpipe_init_notif_group_and_buckets(gxio_mpipe_context_t *context, 196 unsigned int group, 197 unsigned int ring, 198 unsigned int num_rings, 199 unsigned int bucket, 200 unsigned int num_buckets, 201 gxio_mpipe_bucket_mode_t mode) 202{ 203 int i; 204 int result; 205 206 gxio_mpipe_bucket_info_t bucket_info = { { 207 .group = group, 208 .mode = mode, 209 } 210 }; 211 212 gxio_mpipe_notif_group_bits_t bits = { {0} }; 213 214 for (i = 0; i < num_rings; i++) 215 gxio_mpipe_notif_group_add_ring(&bits, ring + i); 216 217 result = gxio_mpipe_init_notif_group(context, group, bits); 218 if (result != 0) 219 return result; 220 221 for (i = 0; i < num_buckets; i++) { 222 bucket_info.notifring = ring + (i % num_rings); 223 224 result = gxio_mpipe_init_bucket(context, bucket + i, 225 bucket_info); 226 if (result != 0) 227 return result; 228 } 229 230 return 0; 231} 232 233EXPORT_SYMBOL_GPL(gxio_mpipe_init_notif_group_and_buckets); 234 235int gxio_mpipe_init_edma_ring(gxio_mpipe_context_t *context, 236 unsigned int ring, unsigned int channel, 237 void *mem, size_t mem_size, 238 unsigned int mem_flags) 239{ 240 memset(mem, 0, mem_size); 241 242 return gxio_mpipe_init_edma_ring_aux(context, mem, mem_size, mem_flags, 243 ring, channel); 244} 245 246EXPORT_SYMBOL_GPL(gxio_mpipe_init_edma_ring); 247 248void gxio_mpipe_rules_init(gxio_mpipe_rules_t *rules, 249 gxio_mpipe_context_t *context) 250{ 251 rules->context = context; 252 memset(&rules->list, 0, sizeof(rules->list)); 253} 254 255EXPORT_SYMBOL_GPL(gxio_mpipe_rules_init); 256 257int gxio_mpipe_rules_begin(gxio_mpipe_rules_t *rules, 258 unsigned int bucket, unsigned int num_buckets, 259 gxio_mpipe_rules_stacks_t *stacks) 260{ 261 int i; 262 int stack = 255; 263 264 gxio_mpipe_rules_list_t *list = &rules->list; 265 266 /* Current rule. */ 267 gxio_mpipe_rules_rule_t *rule = 268 (gxio_mpipe_rules_rule_t *) (list->rules + list->head); 269 270 unsigned int head = list->tail; 271 272 /* 273 * Align next rule properly. 274 *Note that "dmacs_and_vlans" will also be aligned. 275 */ 276 unsigned int pad = 0; 277 while (((head + pad) % __alignof__(gxio_mpipe_rules_rule_t)) != 0) 278 pad++; 279 280 /* 281 * Verify room. 282 * ISSUE: Mark rules as broken on error? 283 */ 284 if (head + pad + sizeof(*rule) >= sizeof(list->rules)) 285 return GXIO_MPIPE_ERR_RULES_FULL; 286 287 /* Verify num_buckets is a power of 2. */ 288 if (__builtin_popcount(num_buckets) != 1) 289 return GXIO_MPIPE_ERR_RULES_INVALID; 290 291 /* Add padding to previous rule. */ 292 rule->size += pad; 293 294 /* Start a new rule. */ 295 list->head = head + pad; 296 297 rule = (gxio_mpipe_rules_rule_t *) (list->rules + list->head); 298 299 /* Default some values. */ 300 rule->headroom = 2; 301 rule->tailroom = 0; 302 rule->capacity = 16384; 303 304 /* Save the bucket info. */ 305 rule->bucket_mask = num_buckets - 1; 306 rule->bucket_first = bucket; 307 308 for (i = 8 - 1; i >= 0; i--) { 309 int maybe = 310 stacks ? stacks->stacks[i] : rules->context->__stacks. 311 stacks[i]; 312 if (maybe != 255) 313 stack = maybe; 314 rule->stacks.stacks[i] = stack; 315 } 316 317 if (stack == 255) 318 return GXIO_MPIPE_ERR_RULES_INVALID; 319 320 /* NOTE: Only entries at the end of the array can be 255. */ 321 for (i = 8 - 1; i > 0; i--) { 322 if (rule->stacks.stacks[i] == 255) { 323 rule->stacks.stacks[i] = stack; 324 rule->capacity = 325 gxio_mpipe_buffer_size_enum_to_buffer_size(i - 326 1); 327 } 328 } 329 330 rule->size = sizeof(*rule); 331 list->tail = list->head + rule->size; 332 333 return 0; 334} 335 336EXPORT_SYMBOL_GPL(gxio_mpipe_rules_begin); 337 338int gxio_mpipe_rules_add_channel(gxio_mpipe_rules_t *rules, 339 unsigned int channel) 340{ 341 gxio_mpipe_rules_list_t *list = &rules->list; 342 343 gxio_mpipe_rules_rule_t *rule = 344 (gxio_mpipe_rules_rule_t *) (list->rules + list->head); 345 346 /* Verify channel. */ 347 if (channel >= 32) 348 return GXIO_MPIPE_ERR_RULES_INVALID; 349 350 /* Verify begun. */ 351 if (list->tail == 0) 352 return GXIO_MPIPE_ERR_RULES_EMPTY; 353 354 rule->channel_bits |= (1UL << channel); 355 356 return 0; 357} 358 359EXPORT_SYMBOL_GPL(gxio_mpipe_rules_add_channel); 360 361int gxio_mpipe_rules_set_headroom(gxio_mpipe_rules_t *rules, uint8_t headroom) 362{ 363 gxio_mpipe_rules_list_t *list = &rules->list; 364 365 gxio_mpipe_rules_rule_t *rule = 366 (gxio_mpipe_rules_rule_t *) (list->rules + list->head); 367 368 /* Verify begun. */ 369 if (list->tail == 0) 370 return GXIO_MPIPE_ERR_RULES_EMPTY; 371 372 rule->headroom = headroom; 373 374 return 0; 375} 376 377EXPORT_SYMBOL_GPL(gxio_mpipe_rules_set_headroom); 378 379int gxio_mpipe_rules_commit(gxio_mpipe_rules_t *rules) 380{ 381 gxio_mpipe_rules_list_t *list = &rules->list; 382 unsigned int size = 383 offsetof(gxio_mpipe_rules_list_t, rules) + list->tail; 384 return gxio_mpipe_commit_rules(rules->context, list, size); 385} 386 387EXPORT_SYMBOL_GPL(gxio_mpipe_rules_commit); 388 389int gxio_mpipe_iqueue_init(gxio_mpipe_iqueue_t *iqueue, 390 gxio_mpipe_context_t *context, 391 unsigned int ring, 392 void *mem, size_t mem_size, unsigned int mem_flags) 393{ 394 /* The init call below will verify that "mem_size" is legal. */ 395 unsigned int num_entries = mem_size / sizeof(gxio_mpipe_idesc_t); 396 397 iqueue->context = context; 398 iqueue->idescs = (gxio_mpipe_idesc_t *)mem; 399 iqueue->ring = ring; 400 iqueue->num_entries = num_entries; 401 iqueue->mask_num_entries = num_entries - 1; 402 iqueue->log2_num_entries = __builtin_ctz(num_entries); 403 iqueue->head = 1; 404#ifdef __BIG_ENDIAN__ 405 iqueue->swapped = 0; 406#endif 407 408 /* Initialize the "tail". */ 409 __gxio_mmio_write(mem, iqueue->head); 410 411 return gxio_mpipe_init_notif_ring(context, ring, mem, mem_size, 412 mem_flags); 413} 414 415EXPORT_SYMBOL_GPL(gxio_mpipe_iqueue_init); 416 417int gxio_mpipe_equeue_init(gxio_mpipe_equeue_t *equeue, 418 gxio_mpipe_context_t *context, 419 unsigned int ering, 420 unsigned int channel, 421 void *mem, unsigned int mem_size, 422 unsigned int mem_flags) 423{ 424 /* The init call below will verify that "mem_size" is legal. */ 425 unsigned int num_entries = mem_size / sizeof(gxio_mpipe_edesc_t); 426 427 /* Offset used to read number of completed commands. */ 428 MPIPE_EDMA_POST_REGION_ADDR_t offset; 429 430 int result = gxio_mpipe_init_edma_ring(context, ering, channel, 431 mem, mem_size, mem_flags); 432 if (result < 0) 433 return result; 434 435 memset(equeue, 0, sizeof(*equeue)); 436 437 offset.word = 0; 438 offset.region = 439 MPIPE_MMIO_ADDR__REGION_VAL_EDMA - 440 MPIPE_MMIO_ADDR__REGION_VAL_IDMA; 441 offset.ring = ering; 442 443 __gxio_dma_queue_init(&equeue->dma_queue, 444 context->mmio_fast_base + offset.word, 445 num_entries); 446 equeue->edescs = mem; 447 equeue->mask_num_entries = num_entries - 1; 448 equeue->log2_num_entries = __builtin_ctz(num_entries); 449 equeue->context = context; 450 equeue->ering = ering; 451 equeue->channel = channel; 452 453 return 0; 454} 455 456EXPORT_SYMBOL_GPL(gxio_mpipe_equeue_init); 457 458int gxio_mpipe_set_timestamp(gxio_mpipe_context_t *context, 459 const struct timespec64 *ts) 460{ 461 cycles_t cycles = get_cycles(); 462 return gxio_mpipe_set_timestamp_aux(context, (uint64_t)ts->tv_sec, 463 (uint64_t)ts->tv_nsec, 464 (uint64_t)cycles); 465} 466EXPORT_SYMBOL_GPL(gxio_mpipe_set_timestamp); 467 468int gxio_mpipe_get_timestamp(gxio_mpipe_context_t *context, 469 struct timespec64 *ts) 470{ 471 int ret; 472 cycles_t cycles_prev, cycles_now, clock_rate; 473 cycles_prev = get_cycles(); 474 ret = gxio_mpipe_get_timestamp_aux(context, (uint64_t *)&ts->tv_sec, 475 (uint64_t *)&ts->tv_nsec, 476 (uint64_t *)&cycles_now); 477 if (ret < 0) { 478 return ret; 479 } 480 481 clock_rate = get_clock_rate(); 482 ts->tv_nsec -= (cycles_now - cycles_prev) * 1000000000LL / clock_rate; 483 if (ts->tv_nsec < 0) { 484 ts->tv_nsec += 1000000000LL; 485 ts->tv_sec -= 1; 486 } 487 return ret; 488} 489EXPORT_SYMBOL_GPL(gxio_mpipe_get_timestamp); 490 491int gxio_mpipe_adjust_timestamp(gxio_mpipe_context_t *context, int64_t delta) 492{ 493 return gxio_mpipe_adjust_timestamp_aux(context, delta); 494} 495EXPORT_SYMBOL_GPL(gxio_mpipe_adjust_timestamp); 496 497/* Get our internal context used for link name access. This context is 498 * special in that it is not associated with an mPIPE service domain. 499 */ 500static gxio_mpipe_context_t *_gxio_get_link_context(void) 501{ 502 static gxio_mpipe_context_t context; 503 static gxio_mpipe_context_t *contextp; 504 static int tried_open = 0; 505 static DEFINE_MUTEX(mutex); 506 507 mutex_lock(&mutex); 508 509 if (!tried_open) { 510 int i = 0; 511 tried_open = 1; 512 513 /* 514 * "4" here is the maximum possible number of mPIPE shims; it's 515 * an exaggeration but we shouldn't ever go beyond 2 anyway. 516 */ 517 for (i = 0; i < 4; i++) { 518 char file[80]; 519 520 snprintf(file, sizeof(file), "mpipe/%d/iorpc_info", i); 521 context.fd = hv_dev_open((HV_VirtAddr) file, 0); 522 if (context.fd < 0) 523 continue; 524 525 contextp = &context; 526 break; 527 } 528 } 529 530 mutex_unlock(&mutex); 531 532 return contextp; 533} 534 535int gxio_mpipe_link_instance(const char *link_name) 536{ 537 _gxio_mpipe_link_name_t name; 538 gxio_mpipe_context_t *context = _gxio_get_link_context(); 539 540 if (!context) 541 return GXIO_ERR_NO_DEVICE; 542 543 if (strscpy(name.name, link_name, sizeof(name.name)) == 0) 544 return GXIO_ERR_NO_DEVICE; 545 546 return gxio_mpipe_info_instance_aux(context, name); 547} 548EXPORT_SYMBOL_GPL(gxio_mpipe_link_instance); 549 550int gxio_mpipe_link_enumerate_mac(int idx, char *link_name, uint8_t *link_mac) 551{ 552 int rv; 553 _gxio_mpipe_link_name_t name; 554 _gxio_mpipe_link_mac_t mac; 555 556 gxio_mpipe_context_t *context = _gxio_get_link_context(); 557 if (!context) 558 return GXIO_ERR_NO_DEVICE; 559 560 rv = gxio_mpipe_info_enumerate_aux(context, idx, &name, &mac); 561 if (rv >= 0) { 562 if (strscpy(link_name, name.name, sizeof(name.name)) == 0) 563 return GXIO_ERR_INVAL_MEMORY_SIZE; 564 memcpy(link_mac, mac.mac, sizeof(mac.mac)); 565 } 566 567 return rv; 568} 569 570EXPORT_SYMBOL_GPL(gxio_mpipe_link_enumerate_mac); 571 572int gxio_mpipe_link_open(gxio_mpipe_link_t *link, 573 gxio_mpipe_context_t *context, const char *link_name, 574 unsigned int flags) 575{ 576 _gxio_mpipe_link_name_t name; 577 int rv; 578 579 if (strscpy(name.name, link_name, sizeof(name.name)) == 0) 580 return GXIO_ERR_NO_DEVICE; 581 582 rv = gxio_mpipe_link_open_aux(context, name, flags); 583 if (rv < 0) 584 return rv; 585 586 link->context = context; 587 link->channel = rv >> 8; 588 link->mac = rv & 0xFF; 589 590 return 0; 591} 592 593EXPORT_SYMBOL_GPL(gxio_mpipe_link_open); 594 595int gxio_mpipe_link_close(gxio_mpipe_link_t *link) 596{ 597 return gxio_mpipe_link_close_aux(link->context, link->mac); 598} 599 600EXPORT_SYMBOL_GPL(gxio_mpipe_link_close); 601 602int gxio_mpipe_link_set_attr(gxio_mpipe_link_t *link, uint32_t attr, 603 int64_t val) 604{ 605 return gxio_mpipe_link_set_attr_aux(link->context, link->mac, attr, 606 val); 607} 608 609EXPORT_SYMBOL_GPL(gxio_mpipe_link_set_attr); 610