drivers/crypto/caam/qi.c

/* [<][>][^][v][top][bottom][index][help] */
This source file includes following definitions.
caam_iova_to_virt
caam_qi_enqueue
caam_fq_ern_cb
create_caam_req_fq
empty_retired_fq
kill_fq
empty_caam_fq
caam_drv_ctx_update
caam_drv_ctx_init
qi_cache_alloc
qi_cache_free
caam_qi_poll
caam_drv_ctx_rel
caam_qi_shutdown
cgr_cb
caam_qi_napi_schedule
caam_rsp_fq_dqrr_cb
alloc_rsp_fq_cpu
init_cgr
alloc_rsp_fqs
free_rsp_fqs
caam_qi_init
   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * CAAM/SEC 4.x QI transport/backend driver
   4  * Queue Interface backend functionality
   5  *
   6  * Copyright 2013-2016 Freescale Semiconductor, Inc.
   7  * Copyright 2016-2017, 2019 NXP
   8  */
   9 
  10 #include <linux/cpumask.h>
  11 #include <linux/kthread.h>
  12 #include <soc/fsl/qman.h>
  13 
  14 #include "regs.h"
  15 #include "qi.h"
  16 #include "desc.h"
  17 #include "intern.h"
  18 #include "desc_constr.h"
  19 
  20 #define PREHDR_RSLS_SHIFT       31
  21 #define PREHDR_ABS              BIT(25)
  22 
  23 /*
  24  * Use a reasonable backlog of frames (per CPU) as congestion threshold,
  25  * so that resources used by the in-flight buffers do not become a memory hog.
  26  */
  27 #define MAX_RSP_FQ_BACKLOG_PER_CPU      256
  28 
  29 #define CAAM_QI_ENQUEUE_RETRIES 10000
  30 
  31 #define CAAM_NAPI_WEIGHT        63
  32 
  33 /*
  34  * caam_napi - struct holding CAAM NAPI-related params
  35  * @irqtask: IRQ task for QI backend
  36  * @p: QMan portal
  37  */
  38 struct caam_napi {
  39         struct napi_struct irqtask;
  40         struct qman_portal *p;
  41 };
  42 
  43 /*
  44  * caam_qi_pcpu_priv - percpu private data structure to main list of pending
  45  *                     responses expected on each cpu.
  46  * @caam_napi: CAAM NAPI params
  47  * @net_dev: netdev used by NAPI
  48  * @rsp_fq: response FQ from CAAM
  49  */
  50 struct caam_qi_pcpu_priv {
  51         struct caam_napi caam_napi;
  52         struct net_device net_dev;
  53         struct qman_fq *rsp_fq;
  54 } ____cacheline_aligned;
  55 
  56 static DEFINE_PER_CPU(struct caam_qi_pcpu_priv, pcpu_qipriv);
  57 static DEFINE_PER_CPU(int, last_cpu);
  58 
  59 /*
  60  * caam_qi_priv - CAAM QI backend private params
  61  * @cgr: QMan congestion group
  62  */
  63 struct caam_qi_priv {
  64         struct qman_cgr cgr;
  65 };
  66 
  67 static struct caam_qi_priv qipriv ____cacheline_aligned;
  68 
  69 /*
  70  * This is written by only one core - the one that initialized the CGR - and
  71  * read by multiple cores (all the others).
  72  */
  73 bool caam_congested __read_mostly;
  74 EXPORT_SYMBOL(caam_congested);
  75 
  76 #ifdef CONFIG_DEBUG_FS
  77 /*
  78  * This is a counter for the number of times the congestion group (where all
  79  * the request and response queueus are) reached congestion. Incremented
  80  * each time the congestion callback is called with congested == true.
  81  */
  82 static u64 times_congested;
  83 #endif
  84 
  85 /*
  86  * This is a a cache of buffers, from which the users of CAAM QI driver
  87  * can allocate short (CAAM_QI_MEMCACHE_SIZE) buffers. It's faster than
  88  * doing malloc on the hotpath.
  89  * NOTE: A more elegant solution would be to have some headroom in the frames
  90  *       being processed. This could be added by the dpaa-ethernet driver.
  91  *       This would pose a problem for userspace application processing which
  92  *       cannot know of this limitation. So for now, this will work.
  93  * NOTE: The memcache is SMP-safe. No need to handle spinlocks in-here
  94  */
  95 static struct kmem_cache *qi_cache;
  96 
  97 static void *caam_iova_to_virt(struct iommu_domain *domain,
  98                                dma_addr_t iova_addr)
  99 {
 100         phys_addr_t phys_addr;
 101 
 102         phys_addr = domain ? iommu_iova_to_phys(domain, iova_addr) : iova_addr;
 103 
 104         return phys_to_virt(phys_addr);
 105 }
 106 
 107 int caam_qi_enqueue(struct device *qidev, struct caam_drv_req *req)
 108 {
 109         struct qm_fd fd;
 110         dma_addr_t addr;
 111         int ret;
 112         int num_retries = 0;
 113 
 114         qm_fd_clear_fd(&fd);
 115         qm_fd_set_compound(&fd, qm_sg_entry_get_len(&req->fd_sgt[1]));
 116 
 117         addr = dma_map_single(qidev, req->fd_sgt, sizeof(req->fd_sgt),
 118                               DMA_BIDIRECTIONAL);
 119         if (dma_mapping_error(qidev, addr)) {
 120                 dev_err(qidev, "DMA mapping error for QI enqueue request\n");
 121                 return -EIO;
 122         }
 123         qm_fd_addr_set64(&fd, addr);
 124 
 125         do {
 126                 ret = qman_enqueue(req->drv_ctx->req_fq, &fd);
 127                 if (likely(!ret))
 128                         return 0;
 129 
 130                 if (ret != -EBUSY)
 131                         break;
 132                 num_retries++;
 133         } while (num_retries < CAAM_QI_ENQUEUE_RETRIES);
 134 
 135         dev_err(qidev, "qman_enqueue failed: %d\n", ret);
 136 
 137         return ret;
 138 }
 139 EXPORT_SYMBOL(caam_qi_enqueue);
 140 
 141 static void caam_fq_ern_cb(struct qman_portal *qm, struct qman_fq *fq,
 142                            const union qm_mr_entry *msg)
 143 {
 144         const struct qm_fd *fd;
 145         struct caam_drv_req *drv_req;
 146         struct device *qidev = &(raw_cpu_ptr(&pcpu_qipriv)->net_dev.dev);
 147         struct caam_drv_private *priv = dev_get_drvdata(qidev);
 148 
 149         fd = &msg->ern.fd;
 150 
 151         if (qm_fd_get_format(fd) != qm_fd_compound) {
 152                 dev_err(qidev, "Non-compound FD from CAAM\n");
 153                 return;
 154         }
 155 
 156         drv_req = caam_iova_to_virt(priv->domain, qm_fd_addr_get64(fd));
 157         if (!drv_req) {
 158                 dev_err(qidev,
 159                         "Can't find original request for CAAM response\n");
 160                 return;
 161         }
 162 
 163         dma_unmap_single(drv_req->drv_ctx->qidev, qm_fd_addr(fd),
 164                          sizeof(drv_req->fd_sgt), DMA_BIDIRECTIONAL);
 165 
 166         if (fd->status)
 167                 drv_req->cbk(drv_req, be32_to_cpu(fd->status));
 168         else
 169                 drv_req->cbk(drv_req, JRSTA_SSRC_QI);
 170 }
 171 
 172 static struct qman_fq *create_caam_req_fq(struct device *qidev,
 173                                           struct qman_fq *rsp_fq,
 174                                           dma_addr_t hwdesc,
 175                                           int fq_sched_flag)
 176 {
 177         int ret;
 178         struct qman_fq *req_fq;
 179         struct qm_mcc_initfq opts;
 180 
 181         req_fq = kzalloc(sizeof(*req_fq), GFP_ATOMIC);
 182         if (!req_fq)
 183                 return ERR_PTR(-ENOMEM);
 184 
 185         req_fq->cb.ern = caam_fq_ern_cb;
 186         req_fq->cb.fqs = NULL;
 187 
 188         ret = qman_create_fq(0, QMAN_FQ_FLAG_DYNAMIC_FQID |
 189                                 QMAN_FQ_FLAG_TO_DCPORTAL, req_fq);
 190         if (ret) {
 191                 dev_err(qidev, "Failed to create session req FQ\n");
 192                 goto create_req_fq_fail;
 193         }
 194 
 195         memset(&opts, 0, sizeof(opts));
 196         opts.we_mask = cpu_to_be16(QM_INITFQ_WE_FQCTRL | QM_INITFQ_WE_DESTWQ |
 197                                    QM_INITFQ_WE_CONTEXTB |
 198                                    QM_INITFQ_WE_CONTEXTA | QM_INITFQ_WE_CGID);
 199         opts.fqd.fq_ctrl = cpu_to_be16(QM_FQCTRL_CPCSTASH | QM_FQCTRL_CGE);
 200         qm_fqd_set_destwq(&opts.fqd, qm_channel_caam, 2);
 201         opts.fqd.context_b = cpu_to_be32(qman_fq_fqid(rsp_fq));
 202         qm_fqd_context_a_set64(&opts.fqd, hwdesc);
 203         opts.fqd.cgid = qipriv.cgr.cgrid;
 204 
 205         ret = qman_init_fq(req_fq, fq_sched_flag, &opts);
 206         if (ret) {
 207                 dev_err(qidev, "Failed to init session req FQ\n");
 208                 goto init_req_fq_fail;
 209         }
 210 
 211         dev_dbg(qidev, "Allocated request FQ %u for CPU %u\n", req_fq->fqid,
 212                 smp_processor_id());
 213         return req_fq;
 214 
 215 init_req_fq_fail:
 216         qman_destroy_fq(req_fq);
 217 create_req_fq_fail:
 218         kfree(req_fq);
 219         return ERR_PTR(ret);
 220 }
 221 
 222 static int empty_retired_fq(struct device *qidev, struct qman_fq *fq)
 223 {
 224         int ret;
 225 
 226         ret = qman_volatile_dequeue(fq, QMAN_VOLATILE_FLAG_WAIT_INT |
 227                                     QMAN_VOLATILE_FLAG_FINISH,
 228                                     QM_VDQCR_PRECEDENCE_VDQCR |
 229                                     QM_VDQCR_NUMFRAMES_TILLEMPTY);
 230         if (ret) {
 231                 dev_err(qidev, "Volatile dequeue fail for FQ: %u\n", fq->fqid);
 232                 return ret;
 233         }
 234 
 235         do {
 236                 struct qman_portal *p;
 237 
 238                 p = qman_get_affine_portal(smp_processor_id());
 239                 qman_p_poll_dqrr(p, 16);
 240         } while (fq->flags & QMAN_FQ_STATE_NE);
 241 
 242         return 0;
 243 }
 244 
 245 static int kill_fq(struct device *qidev, struct qman_fq *fq)
 246 {
 247         u32 flags;
 248         int ret;
 249 
 250         ret = qman_retire_fq(fq, &flags);
 251         if (ret < 0) {
 252                 dev_err(qidev, "qman_retire_fq failed: %d\n", ret);
 253                 return ret;
 254         }
 255 
 256         if (!ret)
 257                 goto empty_fq;
 258 
 259         /* Async FQ retirement condition */
 260         if (ret == 1) {
 261                 /* Retry till FQ gets in retired state */
 262                 do {
 263                         msleep(20);
 264                 } while (fq->state != qman_fq_state_retired);
 265 
 266                 WARN_ON(fq->flags & QMAN_FQ_STATE_BLOCKOOS);
 267                 WARN_ON(fq->flags & QMAN_FQ_STATE_ORL);
 268         }
 269 
 270 empty_fq:
 271         if (fq->flags & QMAN_FQ_STATE_NE) {
 272                 ret = empty_retired_fq(qidev, fq);
 273                 if (ret) {
 274                         dev_err(qidev, "empty_retired_fq fail for FQ: %u\n",
 275                                 fq->fqid);
 276                         return ret;
 277                 }
 278         }
 279 
 280         ret = qman_oos_fq(fq);
 281         if (ret)
 282                 dev_err(qidev, "OOS of FQID: %u failed\n", fq->fqid);
 283 
 284         qman_destroy_fq(fq);
 285         kfree(fq);
 286 
 287         return ret;
 288 }
 289 
 290 static int empty_caam_fq(struct qman_fq *fq)
 291 {
 292         int ret;
 293         struct qm_mcr_queryfq_np np;
 294 
 295         /* Wait till the older CAAM FQ get empty */
 296         do {
 297                 ret = qman_query_fq_np(fq, &np);
 298                 if (ret)
 299                         return ret;
 300 
 301                 if (!qm_mcr_np_get(&np, frm_cnt))
 302                         break;
 303 
 304                 msleep(20);
 305         } while (1);
 306 
 307         /*
 308          * Give extra time for pending jobs from this FQ in holding tanks
 309          * to get processed
 310          */
 311         msleep(20);
 312         return 0;
 313 }
 314 
 315 int caam_drv_ctx_update(struct caam_drv_ctx *drv_ctx, u32 *sh_desc)
 316 {
 317         int ret;
 318         u32 num_words;
 319         struct qman_fq *new_fq, *old_fq;
 320         struct device *qidev = drv_ctx->qidev;
 321 
 322         num_words = desc_len(sh_desc);
 323         if (num_words > MAX_SDLEN) {
 324                 dev_err(qidev, "Invalid descriptor len: %d words\n", num_words);
 325                 return -EINVAL;
 326         }
 327 
 328         /* Note down older req FQ */
 329         old_fq = drv_ctx->req_fq;
 330 
 331         /* Create a new req FQ in parked state */
 332         new_fq = create_caam_req_fq(drv_ctx->qidev, drv_ctx->rsp_fq,
 333                                     drv_ctx->context_a, 0);
 334         if (IS_ERR(new_fq)) {
 335                 dev_err(qidev, "FQ allocation for shdesc update failed\n");
 336                 return PTR_ERR(new_fq);
 337         }
 338 
 339         /* Hook up new FQ to context so that new requests keep queuing */
 340         drv_ctx->req_fq = new_fq;
 341 
 342         /* Empty and remove the older FQ */
 343         ret = empty_caam_fq(old_fq);
 344         if (ret) {
 345                 dev_err(qidev, "Old CAAM FQ empty failed: %d\n", ret);
 346 
 347                 /* We can revert to older FQ */
 348                 drv_ctx->req_fq = old_fq;
 349 
 350                 if (kill_fq(qidev, new_fq))
 351                         dev_warn(qidev, "New CAAM FQ kill failed\n");
 352 
 353                 return ret;
 354         }
 355 
 356         /*
 357          * Re-initialise pre-header. Set RSLS and SDLEN.
 358          * Update the shared descriptor for driver context.
 359          */
 360         drv_ctx->prehdr[0] = cpu_to_caam32((1 << PREHDR_RSLS_SHIFT) |
 361                                            num_words);
 362         drv_ctx->prehdr[1] = cpu_to_caam32(PREHDR_ABS);
 363         memcpy(drv_ctx->sh_desc, sh_desc, desc_bytes(sh_desc));
 364         dma_sync_single_for_device(qidev, drv_ctx->context_a,
 365                                    sizeof(drv_ctx->sh_desc) +
 366                                    sizeof(drv_ctx->prehdr),
 367                                    DMA_BIDIRECTIONAL);
 368 
 369         /* Put the new FQ in scheduled state */
 370         ret = qman_schedule_fq(new_fq);
 371         if (ret) {
 372                 dev_err(qidev, "Fail to sched new CAAM FQ, ecode = %d\n", ret);
 373 
 374                 /*
 375                  * We can kill new FQ and revert to old FQ.
 376                  * Since the desc is already modified, it is success case
 377                  */
 378 
 379                 drv_ctx->req_fq = old_fq;
 380 
 381                 if (kill_fq(qidev, new_fq))
 382                         dev_warn(qidev, "New CAAM FQ kill failed\n");
 383         } else if (kill_fq(qidev, old_fq)) {
 384                 dev_warn(qidev, "Old CAAM FQ kill failed\n");
 385         }
 386 
 387         return 0;
 388 }
 389 EXPORT_SYMBOL(caam_drv_ctx_update);
 390 
 391 struct caam_drv_ctx *caam_drv_ctx_init(struct device *qidev,
 392                                        int *cpu,
 393                                        u32 *sh_desc)
 394 {
 395         size_t size;
 396         u32 num_words;
 397         dma_addr_t hwdesc;
 398         struct caam_drv_ctx *drv_ctx;
 399         const cpumask_t *cpus = qman_affine_cpus();
 400 
 401         num_words = desc_len(sh_desc);
 402         if (num_words > MAX_SDLEN) {
 403                 dev_err(qidev, "Invalid descriptor len: %d words\n",
 404                         num_words);
 405                 return ERR_PTR(-EINVAL);
 406         }
 407 
 408         drv_ctx = kzalloc(sizeof(*drv_ctx), GFP_ATOMIC);
 409         if (!drv_ctx)
 410                 return ERR_PTR(-ENOMEM);
 411 
 412         /*
 413          * Initialise pre-header - set RSLS and SDLEN - and shared descriptor
 414          * and dma-map them.
 415          */
 416         drv_ctx->prehdr[0] = cpu_to_caam32((1 << PREHDR_RSLS_SHIFT) |
 417                                            num_words);
 418         drv_ctx->prehdr[1] = cpu_to_caam32(PREHDR_ABS);
 419         memcpy(drv_ctx->sh_desc, sh_desc, desc_bytes(sh_desc));
 420         size = sizeof(drv_ctx->prehdr) + sizeof(drv_ctx->sh_desc);
 421         hwdesc = dma_map_single(qidev, drv_ctx->prehdr, size,
 422                                 DMA_BIDIRECTIONAL);
 423         if (dma_mapping_error(qidev, hwdesc)) {
 424                 dev_err(qidev, "DMA map error for preheader + shdesc\n");
 425                 kfree(drv_ctx);
 426                 return ERR_PTR(-ENOMEM);
 427         }
 428         drv_ctx->context_a = hwdesc;
 429 
 430         /* If given CPU does not own the portal, choose another one that does */
 431         if (!cpumask_test_cpu(*cpu, cpus)) {
 432                 int *pcpu = &get_cpu_var(last_cpu);
 433 
 434                 *pcpu = cpumask_next(*pcpu, cpus);
 435                 if (*pcpu >= nr_cpu_ids)
 436                         *pcpu = cpumask_first(cpus);
 437                 *cpu = *pcpu;
 438 
 439                 put_cpu_var(last_cpu);
 440         }
 441         drv_ctx->cpu = *cpu;
 442 
 443         /* Find response FQ hooked with this CPU */
 444         drv_ctx->rsp_fq = per_cpu(pcpu_qipriv.rsp_fq, drv_ctx->cpu);
 445 
 446         /* Attach request FQ */
 447         drv_ctx->req_fq = create_caam_req_fq(qidev, drv_ctx->rsp_fq, hwdesc,
 448                                              QMAN_INITFQ_FLAG_SCHED);
 449         if (IS_ERR(drv_ctx->req_fq)) {
 450                 dev_err(qidev, "create_caam_req_fq failed\n");
 451                 dma_unmap_single(qidev, hwdesc, size, DMA_BIDIRECTIONAL);
 452                 kfree(drv_ctx);
 453                 return ERR_PTR(-ENOMEM);
 454         }
 455 
 456         drv_ctx->qidev = qidev;
 457         return drv_ctx;
 458 }
 459 EXPORT_SYMBOL(caam_drv_ctx_init);
 460 
 461 void *qi_cache_alloc(gfp_t flags)
 462 {
 463         return kmem_cache_alloc(qi_cache, flags);
 464 }
 465 EXPORT_SYMBOL(qi_cache_alloc);
 466 
 467 void qi_cache_free(void *obj)
 468 {
 469         kmem_cache_free(qi_cache, obj);
 470 }
 471 EXPORT_SYMBOL(qi_cache_free);
 472 
 473 static int caam_qi_poll(struct napi_struct *napi, int budget)
 474 {
 475         struct caam_napi *np = container_of(napi, struct caam_napi, irqtask);
 476 
 477         int cleaned = qman_p_poll_dqrr(np->p, budget);
 478 
 479         if (cleaned < budget) {
 480                 napi_complete(napi);
 481                 qman_p_irqsource_add(np->p, QM_PIRQ_DQRI);
 482         }
 483 
 484         return cleaned;
 485 }
 486 
 487 void caam_drv_ctx_rel(struct caam_drv_ctx *drv_ctx)
 488 {
 489         if (IS_ERR_OR_NULL(drv_ctx))
 490                 return;
 491 
 492         /* Remove request FQ */
 493         if (kill_fq(drv_ctx->qidev, drv_ctx->req_fq))
 494                 dev_err(drv_ctx->qidev, "Crypto session req FQ kill failed\n");
 495 
 496         dma_unmap_single(drv_ctx->qidev, drv_ctx->context_a,
 497                          sizeof(drv_ctx->sh_desc) + sizeof(drv_ctx->prehdr),
 498                          DMA_BIDIRECTIONAL);
 499         kfree(drv_ctx);
 500 }
 501 EXPORT_SYMBOL(caam_drv_ctx_rel);
 502 
 503 void caam_qi_shutdown(struct device *qidev)
 504 {
 505         int i;
 506         struct caam_qi_priv *priv = &qipriv;
 507         const cpumask_t *cpus = qman_affine_cpus();
 508 
 509         for_each_cpu(i, cpus) {
 510                 struct napi_struct *irqtask;
 511 
 512                 irqtask = &per_cpu_ptr(&pcpu_qipriv.caam_napi, i)->irqtask;
 513                 napi_disable(irqtask);
 514                 netif_napi_del(irqtask);
 515 
 516                 if (kill_fq(qidev, per_cpu(pcpu_qipriv.rsp_fq, i)))
 517                         dev_err(qidev, "Rsp FQ kill failed, cpu: %d\n", i);
 518         }
 519 
 520         qman_delete_cgr_safe(&priv->cgr);
 521         qman_release_cgrid(priv->cgr.cgrid);
 522 
 523         kmem_cache_destroy(qi_cache);
 524 }
 525 
 526 static void cgr_cb(struct qman_portal *qm, struct qman_cgr *cgr, int congested)
 527 {
 528         caam_congested = congested;
 529 
 530         if (congested) {
 531 #ifdef CONFIG_DEBUG_FS
 532                 times_congested++;
 533 #endif
 534                 pr_debug_ratelimited("CAAM entered congestion\n");
 535 
 536         } else {
 537                 pr_debug_ratelimited("CAAM exited congestion\n");
 538         }
 539 }
 540 
 541 static int caam_qi_napi_schedule(struct qman_portal *p, struct caam_napi *np)
 542 {
 543         /*
 544          * In case of threaded ISR, for RT kernels in_irq() does not return
 545          * appropriate value, so use in_serving_softirq to distinguish between
 546          * softirq and irq contexts.
 547          */
 548         if (unlikely(in_irq() || !in_serving_softirq())) {
 549                 /* Disable QMan IRQ source and invoke NAPI */
 550                 qman_p_irqsource_remove(p, QM_PIRQ_DQRI);
 551                 np->p = p;
 552                 napi_schedule(&np->irqtask);
 553                 return 1;
 554         }
 555         return 0;
 556 }
 557 
 558 static enum qman_cb_dqrr_result caam_rsp_fq_dqrr_cb(struct qman_portal *p,
 559                                                     struct qman_fq *rsp_fq,
 560                                                     const struct qm_dqrr_entry *dqrr)
 561 {
 562         struct caam_napi *caam_napi = raw_cpu_ptr(&pcpu_qipriv.caam_napi);
 563         struct caam_drv_req *drv_req;
 564         const struct qm_fd *fd;
 565         struct device *qidev = &(raw_cpu_ptr(&pcpu_qipriv)->net_dev.dev);
 566         struct caam_drv_private *priv = dev_get_drvdata(qidev);
 567         u32 status;
 568 
 569         if (caam_qi_napi_schedule(p, caam_napi))
 570                 return qman_cb_dqrr_stop;
 571 
 572         fd = &dqrr->fd;
 573         status = be32_to_cpu(fd->status);
 574         if (unlikely(status)) {
 575                 u32 ssrc = status & JRSTA_SSRC_MASK;
 576                 u8 err_id = status & JRSTA_CCBERR_ERRID_MASK;
 577 
 578                 if (ssrc != JRSTA_SSRC_CCB_ERROR ||
 579                     err_id != JRSTA_CCBERR_ERRID_ICVCHK)
 580                         dev_err_ratelimited(qidev,
 581                                             "Error: %#x in CAAM response FD\n",
 582                                             status);
 583         }
 584 
 585         if (unlikely(qm_fd_get_format(fd) != qm_fd_compound)) {
 586                 dev_err(qidev, "Non-compound FD from CAAM\n");
 587                 return qman_cb_dqrr_consume;
 588         }
 589 
 590         drv_req = caam_iova_to_virt(priv->domain, qm_fd_addr_get64(fd));
 591         if (unlikely(!drv_req)) {
 592                 dev_err(qidev,
 593                         "Can't find original request for caam response\n");
 594                 return qman_cb_dqrr_consume;
 595         }
 596 
 597         dma_unmap_single(drv_req->drv_ctx->qidev, qm_fd_addr(fd),
 598                          sizeof(drv_req->fd_sgt), DMA_BIDIRECTIONAL);
 599 
 600         drv_req->cbk(drv_req, status);
 601         return qman_cb_dqrr_consume;
 602 }
 603 
 604 static int alloc_rsp_fq_cpu(struct device *qidev, unsigned int cpu)
 605 {
 606         struct qm_mcc_initfq opts;
 607         struct qman_fq *fq;
 608         int ret;
 609 
 610         fq = kzalloc(sizeof(*fq), GFP_KERNEL | GFP_DMA);
 611         if (!fq)
 612                 return -ENOMEM;
 613 
 614         fq->cb.dqrr = caam_rsp_fq_dqrr_cb;
 615 
 616         ret = qman_create_fq(0, QMAN_FQ_FLAG_NO_ENQUEUE |
 617                              QMAN_FQ_FLAG_DYNAMIC_FQID, fq);
 618         if (ret) {
 619                 dev_err(qidev, "Rsp FQ create failed\n");
 620                 kfree(fq);
 621                 return -ENODEV;
 622         }
 623 
 624         memset(&opts, 0, sizeof(opts));
 625         opts.we_mask = cpu_to_be16(QM_INITFQ_WE_FQCTRL | QM_INITFQ_WE_DESTWQ |
 626                                    QM_INITFQ_WE_CONTEXTB |
 627                                    QM_INITFQ_WE_CONTEXTA | QM_INITFQ_WE_CGID);
 628         opts.fqd.fq_ctrl = cpu_to_be16(QM_FQCTRL_CTXASTASHING |
 629                                        QM_FQCTRL_CPCSTASH | QM_FQCTRL_CGE);
 630         qm_fqd_set_destwq(&opts.fqd, qman_affine_channel(cpu), 3);
 631         opts.fqd.cgid = qipriv.cgr.cgrid;
 632         opts.fqd.context_a.stashing.exclusive = QM_STASHING_EXCL_CTX |
 633                                                 QM_STASHING_EXCL_DATA;
 634         qm_fqd_set_stashing(&opts.fqd, 0, 1, 1);
 635 
 636         ret = qman_init_fq(fq, QMAN_INITFQ_FLAG_SCHED, &opts);
 637         if (ret) {
 638                 dev_err(qidev, "Rsp FQ init failed\n");
 639                 kfree(fq);
 640                 return -ENODEV;
 641         }
 642 
 643         per_cpu(pcpu_qipriv.rsp_fq, cpu) = fq;
 644 
 645         dev_dbg(qidev, "Allocated response FQ %u for CPU %u", fq->fqid, cpu);
 646         return 0;
 647 }
 648 
 649 static int init_cgr(struct device *qidev)
 650 {
 651         int ret;
 652         struct qm_mcc_initcgr opts;
 653         const u64 val = (u64)cpumask_weight(qman_affine_cpus()) *
 654                         MAX_RSP_FQ_BACKLOG_PER_CPU;
 655 
 656         ret = qman_alloc_cgrid(&qipriv.cgr.cgrid);
 657         if (ret) {
 658                 dev_err(qidev, "CGR alloc failed for rsp FQs: %d\n", ret);
 659                 return ret;
 660         }
 661 
 662         qipriv.cgr.cb = cgr_cb;
 663         memset(&opts, 0, sizeof(opts));
 664         opts.we_mask = cpu_to_be16(QM_CGR_WE_CSCN_EN | QM_CGR_WE_CS_THRES |
 665                                    QM_CGR_WE_MODE);
 666         opts.cgr.cscn_en = QM_CGR_EN;
 667         opts.cgr.mode = QMAN_CGR_MODE_FRAME;
 668         qm_cgr_cs_thres_set64(&opts.cgr.cs_thres, val, 1);
 669 
 670         ret = qman_create_cgr(&qipriv.cgr, QMAN_CGR_FLAG_USE_INIT, &opts);
 671         if (ret) {
 672                 dev_err(qidev, "Error %d creating CAAM CGRID: %u\n", ret,
 673                         qipriv.cgr.cgrid);
 674                 return ret;
 675         }
 676 
 677         dev_dbg(qidev, "Congestion threshold set to %llu\n", val);
 678         return 0;
 679 }
 680 
 681 static int alloc_rsp_fqs(struct device *qidev)
 682 {
 683         int ret, i;
 684         const cpumask_t *cpus = qman_affine_cpus();
 685 
 686         /*Now create response FQs*/
 687         for_each_cpu(i, cpus) {
 688                 ret = alloc_rsp_fq_cpu(qidev, i);
 689                 if (ret) {
 690                         dev_err(qidev, "CAAM rsp FQ alloc failed, cpu: %u", i);
 691                         return ret;
 692                 }
 693         }
 694 
 695         return 0;
 696 }
 697 
 698 static void free_rsp_fqs(void)
 699 {
 700         int i;
 701         const cpumask_t *cpus = qman_affine_cpus();
 702 
 703         for_each_cpu(i, cpus)
 704                 kfree(per_cpu(pcpu_qipriv.rsp_fq, i));
 705 }
 706 
 707 int caam_qi_init(struct platform_device *caam_pdev)
 708 {
 709         int err, i;
 710         struct device *ctrldev = &caam_pdev->dev, *qidev;
 711         struct caam_drv_private *ctrlpriv;
 712         const cpumask_t *cpus = qman_affine_cpus();
 713 
 714         ctrlpriv = dev_get_drvdata(ctrldev);
 715         qidev = ctrldev;
 716 
 717         /* Initialize the congestion detection */
 718         err = init_cgr(qidev);
 719         if (err) {
 720                 dev_err(qidev, "CGR initialization failed: %d\n", err);
 721                 return err;
 722         }
 723 
 724         /* Initialise response FQs */
 725         err = alloc_rsp_fqs(qidev);
 726         if (err) {
 727                 dev_err(qidev, "Can't allocate CAAM response FQs: %d\n", err);
 728                 free_rsp_fqs();
 729                 return err;
 730         }
 731 
 732         /*
 733          * Enable the NAPI contexts on each of the core which has an affine
 734          * portal.
 735          */
 736         for_each_cpu(i, cpus) {
 737                 struct caam_qi_pcpu_priv *priv = per_cpu_ptr(&pcpu_qipriv, i);
 738                 struct caam_napi *caam_napi = &priv->caam_napi;
 739                 struct napi_struct *irqtask = &caam_napi->irqtask;
 740                 struct net_device *net_dev = &priv->net_dev;
 741 
 742                 net_dev->dev = *qidev;
 743                 INIT_LIST_HEAD(&net_dev->napi_list);
 744 
 745                 netif_napi_add(net_dev, irqtask, caam_qi_poll,
 746                                CAAM_NAPI_WEIGHT);
 747 
 748                 napi_enable(irqtask);
 749         }
 750 
 751         qi_cache = kmem_cache_create("caamqicache", CAAM_QI_MEMCACHE_SIZE, 0,
 752                                      SLAB_CACHE_DMA, NULL);
 753         if (!qi_cache) {
 754                 dev_err(qidev, "Can't allocate CAAM cache\n");
 755                 free_rsp_fqs();
 756                 return -ENOMEM;
 757         }
 758 
 759 #ifdef CONFIG_DEBUG_FS
 760         debugfs_create_file("qi_congested", 0444, ctrlpriv->ctl,
 761                             &times_congested, &caam_fops_u64_ro);
 762 #endif
 763 
 764         ctrlpriv->qi_init = 1;
 765         dev_info(qidev, "Linux CAAM Queue I/F driver initialised\n");
 766         return 0;
 767 }
/* [<][>][^][v][top][bottom][index][help] */
root/drivers/crypto/caam/qi.c

DEFINITIONS
