This source file includes following definitions.
- hisi_zip_config_buf_type
- hisi_zip_config_tag
- hisi_zip_fill_sqe
- hisi_zip_create_qp
- hisi_zip_release_qp
- hisi_zip_ctx_init
- hisi_zip_ctx_exit
- get_extra_field_size
- get_name_field_size
- get_comment_field_size
- __get_gzip_head_size
- hisi_zip_create_req_q
- hisi_zip_release_req_q
- hisi_zip_create_sgl_pool
- hisi_zip_release_sgl_pool
- hisi_zip_remove_req
- hisi_zip_acomp_cb
- hisi_zip_set_acomp_cb
- hisi_zip_acomp_init
- hisi_zip_acomp_exit
- add_comp_head
- get_gzip_head_size
- get_comp_head_size
- hisi_zip_create_req
- hisi_zip_do_work
- hisi_zip_acompress
- hisi_zip_adecompress
- hisi_zip_register_to_crypto
- hisi_zip_unregister_from_crypto
1
2
3 #include <crypto/internal/acompress.h>
4 #include <linux/bitfield.h>
5 #include <linux/dma-mapping.h>
6 #include <linux/scatterlist.h>
7 #include "zip.h"
8
9 #define HZIP_ZLIB_HEAD_SIZE 2
10 #define HZIP_GZIP_HEAD_SIZE 10
11
12 #define GZIP_HEAD_FHCRC_BIT BIT(1)
13 #define GZIP_HEAD_FEXTRA_BIT BIT(2)
14 #define GZIP_HEAD_FNAME_BIT BIT(3)
15 #define GZIP_HEAD_FCOMMENT_BIT BIT(4)
16
17 #define GZIP_HEAD_FLG_SHIFT 3
18 #define GZIP_HEAD_FEXTRA_SHIFT 10
19 #define GZIP_HEAD_FEXTRA_XLEN 2
20 #define GZIP_HEAD_FHCRC_SIZE 2
21
22 #define HZIP_CTX_Q_NUM 2
23 #define HZIP_GZIP_HEAD_BUF 256
24 #define HZIP_ALG_PRIORITY 300
25
26 static const u8 zlib_head[HZIP_ZLIB_HEAD_SIZE] = {0x78, 0x9c};
27 static const u8 gzip_head[HZIP_GZIP_HEAD_SIZE] = {0x1f, 0x8b, 0x08, 0x0, 0x0,
28 0x0, 0x0, 0x0, 0x0, 0x03};
29 enum hisi_zip_alg_type {
30 HZIP_ALG_TYPE_COMP = 0,
31 HZIP_ALG_TYPE_DECOMP = 1,
32 };
33
34 #define COMP_NAME_TO_TYPE(alg_name) \
35 (!strcmp((alg_name), "zlib-deflate") ? HZIP_ALG_TYPE_ZLIB : \
36 !strcmp((alg_name), "gzip") ? HZIP_ALG_TYPE_GZIP : 0) \
37
38 #define TO_HEAD_SIZE(req_type) \
39 (((req_type) == HZIP_ALG_TYPE_ZLIB) ? sizeof(zlib_head) : \
40 ((req_type) == HZIP_ALG_TYPE_GZIP) ? sizeof(gzip_head) : 0) \
41
42 #define TO_HEAD(req_type) \
43 (((req_type) == HZIP_ALG_TYPE_ZLIB) ? zlib_head : \
44 ((req_type) == HZIP_ALG_TYPE_GZIP) ? gzip_head : 0) \
45
46 struct hisi_zip_req {
47 struct acomp_req *req;
48 int sskip;
49 int dskip;
50 struct hisi_acc_hw_sgl *hw_src;
51 struct hisi_acc_hw_sgl *hw_dst;
52 dma_addr_t dma_src;
53 dma_addr_t dma_dst;
54 int req_id;
55 };
56
57 struct hisi_zip_req_q {
58 struct hisi_zip_req *q;
59 unsigned long *req_bitmap;
60 rwlock_t req_lock;
61 u16 size;
62 };
63
64 struct hisi_zip_qp_ctx {
65 struct hisi_qp *qp;
66 struct hisi_zip_sqe zip_sqe;
67 struct hisi_zip_req_q req_q;
68 struct hisi_acc_sgl_pool sgl_pool;
69 struct hisi_zip *zip_dev;
70 struct hisi_zip_ctx *ctx;
71 };
72
73 struct hisi_zip_ctx {
74 #define QPC_COMP 0
75 #define QPC_DECOMP 1
76 struct hisi_zip_qp_ctx qp_ctx[HZIP_CTX_Q_NUM];
77 };
78
79 static void hisi_zip_config_buf_type(struct hisi_zip_sqe *sqe, u8 buf_type)
80 {
81 u32 val;
82
83 val = (sqe->dw9) & ~HZIP_BUF_TYPE_M;
84 val |= FIELD_PREP(HZIP_BUF_TYPE_M, buf_type);
85 sqe->dw9 = val;
86 }
87
88 static void hisi_zip_config_tag(struct hisi_zip_sqe *sqe, u32 tag)
89 {
90 sqe->tag = tag;
91 }
92
93 static void hisi_zip_fill_sqe(struct hisi_zip_sqe *sqe, u8 req_type,
94 dma_addr_t s_addr, dma_addr_t d_addr, u32 slen,
95 u32 dlen, int sskip, int dskip)
96 {
97 memset(sqe, 0, sizeof(struct hisi_zip_sqe));
98
99 sqe->input_data_length = slen - sskip;
100 sqe->dw7 = FIELD_PREP(HZIP_IN_SGE_DATA_OFFSET_M, sskip);
101 sqe->dw8 = FIELD_PREP(HZIP_OUT_SGE_DATA_OFFSET_M, dskip);
102 sqe->dw9 = FIELD_PREP(HZIP_REQ_TYPE_M, req_type);
103 sqe->dest_avail_out = dlen - dskip;
104 sqe->source_addr_l = lower_32_bits(s_addr);
105 sqe->source_addr_h = upper_32_bits(s_addr);
106 sqe->dest_addr_l = lower_32_bits(d_addr);
107 sqe->dest_addr_h = upper_32_bits(d_addr);
108 }
109
110 static int hisi_zip_create_qp(struct hisi_qm *qm, struct hisi_zip_qp_ctx *ctx,
111 int alg_type, int req_type)
112 {
113 struct hisi_qp *qp;
114 int ret;
115
116 qp = hisi_qm_create_qp(qm, alg_type);
117 if (IS_ERR(qp))
118 return PTR_ERR(qp);
119
120 qp->req_type = req_type;
121 qp->qp_ctx = ctx;
122 ctx->qp = qp;
123
124 ret = hisi_qm_start_qp(qp, 0);
125 if (ret < 0)
126 goto err_release_qp;
127
128 return 0;
129
130 err_release_qp:
131 hisi_qm_release_qp(qp);
132 return ret;
133 }
134
135 static void hisi_zip_release_qp(struct hisi_zip_qp_ctx *ctx)
136 {
137 hisi_qm_stop_qp(ctx->qp);
138 hisi_qm_release_qp(ctx->qp);
139 }
140
141 static int hisi_zip_ctx_init(struct hisi_zip_ctx *hisi_zip_ctx, u8 req_type)
142 {
143 struct hisi_zip *hisi_zip;
144 struct hisi_qm *qm;
145 int ret, i, j;
146
147
148 hisi_zip = find_zip_device(cpu_to_node(smp_processor_id()));
149 if (!hisi_zip) {
150 pr_err("Failed to find a proper ZIP device!\n");
151 return -ENODEV;
152 }
153 qm = &hisi_zip->qm;
154
155 for (i = 0; i < HZIP_CTX_Q_NUM; i++) {
156
157 ret = hisi_zip_create_qp(qm, &hisi_zip_ctx->qp_ctx[i], i,
158 req_type);
159 if (ret)
160 goto err;
161
162 hisi_zip_ctx->qp_ctx[i].zip_dev = hisi_zip;
163 }
164
165 return 0;
166 err:
167 for (j = i - 1; j >= 0; j--)
168 hisi_zip_release_qp(&hisi_zip_ctx->qp_ctx[j]);
169
170 return ret;
171 }
172
173 static void hisi_zip_ctx_exit(struct hisi_zip_ctx *hisi_zip_ctx)
174 {
175 int i;
176
177 for (i = 1; i >= 0; i--)
178 hisi_zip_release_qp(&hisi_zip_ctx->qp_ctx[i]);
179 }
180
181 static u16 get_extra_field_size(const u8 *start)
182 {
183 return *((u16 *)start) + GZIP_HEAD_FEXTRA_XLEN;
184 }
185
186 static u32 get_name_field_size(const u8 *start)
187 {
188 return strlen(start) + 1;
189 }
190
191 static u32 get_comment_field_size(const u8 *start)
192 {
193 return strlen(start) + 1;
194 }
195
196 static u32 __get_gzip_head_size(const u8 *src)
197 {
198 u8 head_flg = *(src + GZIP_HEAD_FLG_SHIFT);
199 u32 size = GZIP_HEAD_FEXTRA_SHIFT;
200
201 if (head_flg & GZIP_HEAD_FEXTRA_BIT)
202 size += get_extra_field_size(src + size);
203 if (head_flg & GZIP_HEAD_FNAME_BIT)
204 size += get_name_field_size(src + size);
205 if (head_flg & GZIP_HEAD_FCOMMENT_BIT)
206 size += get_comment_field_size(src + size);
207 if (head_flg & GZIP_HEAD_FHCRC_BIT)
208 size += GZIP_HEAD_FHCRC_SIZE;
209
210 return size;
211 }
212
213 static int hisi_zip_create_req_q(struct hisi_zip_ctx *ctx)
214 {
215 struct hisi_zip_req_q *req_q;
216 int i, ret;
217
218 for (i = 0; i < HZIP_CTX_Q_NUM; i++) {
219 req_q = &ctx->qp_ctx[i].req_q;
220 req_q->size = QM_Q_DEPTH;
221
222 req_q->req_bitmap = kcalloc(BITS_TO_LONGS(req_q->size),
223 sizeof(long), GFP_KERNEL);
224 if (!req_q->req_bitmap) {
225 ret = -ENOMEM;
226 if (i == 0)
227 return ret;
228
229 goto err_free_loop0;
230 }
231 rwlock_init(&req_q->req_lock);
232
233 req_q->q = kcalloc(req_q->size, sizeof(struct hisi_zip_req),
234 GFP_KERNEL);
235 if (!req_q->q) {
236 ret = -ENOMEM;
237 if (i == 0)
238 goto err_free_bitmap;
239 else
240 goto err_free_loop1;
241 }
242 }
243
244 return 0;
245
246 err_free_loop1:
247 kfree(ctx->qp_ctx[QPC_DECOMP].req_q.req_bitmap);
248 err_free_loop0:
249 kfree(ctx->qp_ctx[QPC_COMP].req_q.q);
250 err_free_bitmap:
251 kfree(ctx->qp_ctx[QPC_COMP].req_q.req_bitmap);
252 return ret;
253 }
254
255 static void hisi_zip_release_req_q(struct hisi_zip_ctx *ctx)
256 {
257 int i;
258
259 for (i = 0; i < HZIP_CTX_Q_NUM; i++) {
260 kfree(ctx->qp_ctx[i].req_q.q);
261 kfree(ctx->qp_ctx[i].req_q.req_bitmap);
262 }
263 }
264
265 static int hisi_zip_create_sgl_pool(struct hisi_zip_ctx *ctx)
266 {
267 struct hisi_zip_qp_ctx *tmp;
268 int i, ret;
269
270 for (i = 0; i < HZIP_CTX_Q_NUM; i++) {
271 tmp = &ctx->qp_ctx[i];
272 ret = hisi_acc_create_sgl_pool(&tmp->qp->qm->pdev->dev,
273 &tmp->sgl_pool,
274 QM_Q_DEPTH << 1);
275 if (ret < 0) {
276 if (i == 1)
277 goto err_free_sgl_pool0;
278 return -ENOMEM;
279 }
280 }
281
282 return 0;
283
284 err_free_sgl_pool0:
285 hisi_acc_free_sgl_pool(&ctx->qp_ctx[QPC_COMP].qp->qm->pdev->dev,
286 &ctx->qp_ctx[QPC_COMP].sgl_pool);
287 return -ENOMEM;
288 }
289
290 static void hisi_zip_release_sgl_pool(struct hisi_zip_ctx *ctx)
291 {
292 int i;
293
294 for (i = 0; i < HZIP_CTX_Q_NUM; i++)
295 hisi_acc_free_sgl_pool(&ctx->qp_ctx[i].qp->qm->pdev->dev,
296 &ctx->qp_ctx[i].sgl_pool);
297 }
298
299 static void hisi_zip_remove_req(struct hisi_zip_qp_ctx *qp_ctx,
300 struct hisi_zip_req *req)
301 {
302 struct hisi_zip_req_q *req_q = &qp_ctx->req_q;
303
304 write_lock(&req_q->req_lock);
305 clear_bit(req->req_id, req_q->req_bitmap);
306 memset(req, 0, sizeof(struct hisi_zip_req));
307 write_unlock(&req_q->req_lock);
308 }
309
310 static void hisi_zip_acomp_cb(struct hisi_qp *qp, void *data)
311 {
312 struct hisi_zip_sqe *sqe = data;
313 struct hisi_zip_qp_ctx *qp_ctx = qp->qp_ctx;
314 struct hisi_zip_req_q *req_q = &qp_ctx->req_q;
315 struct hisi_zip_req *req = req_q->q + sqe->tag;
316 struct acomp_req *acomp_req = req->req;
317 struct device *dev = &qp->qm->pdev->dev;
318 u32 status, dlen, head_size;
319 int err = 0;
320
321 status = sqe->dw3 & HZIP_BD_STATUS_M;
322
323 if (status != 0 && status != HZIP_NC_ERR) {
324 dev_err(dev, "%scompress fail in qp%u: %u, output: %u\n",
325 (qp->alg_type == 0) ? "" : "de", qp->qp_id, status,
326 sqe->produced);
327 err = -EIO;
328 }
329 dlen = sqe->produced;
330
331 hisi_acc_sg_buf_unmap(dev, acomp_req->src, req->hw_src);
332 hisi_acc_sg_buf_unmap(dev, acomp_req->dst, req->hw_dst);
333
334 head_size = (qp->alg_type == 0) ? TO_HEAD_SIZE(qp->req_type) : 0;
335 acomp_req->dlen = dlen + head_size;
336
337 if (acomp_req->base.complete)
338 acomp_request_complete(acomp_req, err);
339
340 hisi_zip_remove_req(qp_ctx, req);
341 }
342
343 static void hisi_zip_set_acomp_cb(struct hisi_zip_ctx *ctx,
344 void (*fn)(struct hisi_qp *, void *))
345 {
346 int i;
347
348 for (i = 0; i < HZIP_CTX_Q_NUM; i++)
349 ctx->qp_ctx[i].qp->req_cb = fn;
350 }
351
352 static int hisi_zip_acomp_init(struct crypto_acomp *tfm)
353 {
354 const char *alg_name = crypto_tfm_alg_name(&tfm->base);
355 struct hisi_zip_ctx *ctx = crypto_tfm_ctx(&tfm->base);
356 int ret;
357
358 ret = hisi_zip_ctx_init(ctx, COMP_NAME_TO_TYPE(alg_name));
359 if (ret)
360 return ret;
361
362 ret = hisi_zip_create_req_q(ctx);
363 if (ret)
364 goto err_ctx_exit;
365
366 ret = hisi_zip_create_sgl_pool(ctx);
367 if (ret)
368 goto err_release_req_q;
369
370 hisi_zip_set_acomp_cb(ctx, hisi_zip_acomp_cb);
371
372 return 0;
373
374 err_release_req_q:
375 hisi_zip_release_req_q(ctx);
376 err_ctx_exit:
377 hisi_zip_ctx_exit(ctx);
378 return ret;
379 }
380
381 static void hisi_zip_acomp_exit(struct crypto_acomp *tfm)
382 {
383 struct hisi_zip_ctx *ctx = crypto_tfm_ctx(&tfm->base);
384
385 hisi_zip_set_acomp_cb(ctx, NULL);
386 hisi_zip_release_sgl_pool(ctx);
387 hisi_zip_release_req_q(ctx);
388 hisi_zip_ctx_exit(ctx);
389 }
390
391 static int add_comp_head(struct scatterlist *dst, u8 req_type)
392 {
393 int head_size = TO_HEAD_SIZE(req_type);
394 const u8 *head = TO_HEAD(req_type);
395 int ret;
396
397 ret = sg_copy_from_buffer(dst, sg_nents(dst), head, head_size);
398 if (ret != head_size)
399 return -ENOMEM;
400
401 return head_size;
402 }
403
404 static size_t get_gzip_head_size(struct scatterlist *sgl)
405 {
406 char buf[HZIP_GZIP_HEAD_BUF];
407
408 sg_copy_to_buffer(sgl, sg_nents(sgl), buf, sizeof(buf));
409
410 return __get_gzip_head_size(buf);
411 }
412
413 static size_t get_comp_head_size(struct scatterlist *src, u8 req_type)
414 {
415 switch (req_type) {
416 case HZIP_ALG_TYPE_ZLIB:
417 return TO_HEAD_SIZE(HZIP_ALG_TYPE_ZLIB);
418 case HZIP_ALG_TYPE_GZIP:
419 return get_gzip_head_size(src);
420 default:
421 pr_err("request type does not support!\n");
422 return -EINVAL;
423 }
424 }
425
426 static struct hisi_zip_req *hisi_zip_create_req(struct acomp_req *req,
427 struct hisi_zip_qp_ctx *qp_ctx,
428 size_t head_size, bool is_comp)
429 {
430 struct hisi_zip_req_q *req_q = &qp_ctx->req_q;
431 struct hisi_zip_req *q = req_q->q;
432 struct hisi_zip_req *req_cache;
433 int req_id;
434
435 write_lock(&req_q->req_lock);
436
437 req_id = find_first_zero_bit(req_q->req_bitmap, req_q->size);
438 if (req_id >= req_q->size) {
439 write_unlock(&req_q->req_lock);
440 dev_dbg(&qp_ctx->qp->qm->pdev->dev, "req cache is full!\n");
441 return ERR_PTR(-EBUSY);
442 }
443 set_bit(req_id, req_q->req_bitmap);
444
445 req_cache = q + req_id;
446 req_cache->req_id = req_id;
447 req_cache->req = req;
448
449 if (is_comp) {
450 req_cache->sskip = 0;
451 req_cache->dskip = head_size;
452 } else {
453 req_cache->sskip = head_size;
454 req_cache->dskip = 0;
455 }
456
457 write_unlock(&req_q->req_lock);
458
459 return req_cache;
460 }
461
462 static int hisi_zip_do_work(struct hisi_zip_req *req,
463 struct hisi_zip_qp_ctx *qp_ctx)
464 {
465 struct hisi_zip_sqe *zip_sqe = &qp_ctx->zip_sqe;
466 struct acomp_req *a_req = req->req;
467 struct hisi_qp *qp = qp_ctx->qp;
468 struct device *dev = &qp->qm->pdev->dev;
469 struct hisi_acc_sgl_pool *pool = &qp_ctx->sgl_pool;
470 dma_addr_t input;
471 dma_addr_t output;
472 int ret;
473
474 if (!a_req->src || !a_req->slen || !a_req->dst || !a_req->dlen)
475 return -EINVAL;
476
477 req->hw_src = hisi_acc_sg_buf_map_to_hw_sgl(dev, a_req->src, pool,
478 req->req_id << 1, &input);
479 if (IS_ERR(req->hw_src))
480 return PTR_ERR(req->hw_src);
481 req->dma_src = input;
482
483 req->hw_dst = hisi_acc_sg_buf_map_to_hw_sgl(dev, a_req->dst, pool,
484 (req->req_id << 1) + 1,
485 &output);
486 if (IS_ERR(req->hw_dst)) {
487 ret = PTR_ERR(req->hw_dst);
488 goto err_unmap_input;
489 }
490 req->dma_dst = output;
491
492 hisi_zip_fill_sqe(zip_sqe, qp->req_type, input, output, a_req->slen,
493 a_req->dlen, req->sskip, req->dskip);
494 hisi_zip_config_buf_type(zip_sqe, HZIP_SGL);
495 hisi_zip_config_tag(zip_sqe, req->req_id);
496
497
498 ret = hisi_qp_send(qp, zip_sqe);
499 if (ret < 0)
500 goto err_unmap_output;
501
502 return -EINPROGRESS;
503
504 err_unmap_output:
505 hisi_acc_sg_buf_unmap(dev, a_req->dst, req->hw_dst);
506 err_unmap_input:
507 hisi_acc_sg_buf_unmap(dev, a_req->src, req->hw_src);
508 return ret;
509 }
510
511 static int hisi_zip_acompress(struct acomp_req *acomp_req)
512 {
513 struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm);
514 struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[QPC_COMP];
515 struct hisi_zip_req *req;
516 int head_size;
517 int ret;
518
519
520 head_size = add_comp_head(acomp_req->dst, qp_ctx->qp->req_type);
521 if (head_size < 0)
522 return -ENOMEM;
523
524 req = hisi_zip_create_req(acomp_req, qp_ctx, (size_t)head_size, true);
525 if (IS_ERR(req))
526 return PTR_ERR(req);
527
528 ret = hisi_zip_do_work(req, qp_ctx);
529 if (ret != -EINPROGRESS)
530 hisi_zip_remove_req(qp_ctx, req);
531
532 return ret;
533 }
534
535 static int hisi_zip_adecompress(struct acomp_req *acomp_req)
536 {
537 struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm);
538 struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[QPC_DECOMP];
539 struct hisi_zip_req *req;
540 size_t head_size;
541 int ret;
542
543 head_size = get_comp_head_size(acomp_req->src, qp_ctx->qp->req_type);
544
545 req = hisi_zip_create_req(acomp_req, qp_ctx, head_size, false);
546 if (IS_ERR(req))
547 return PTR_ERR(req);
548
549 ret = hisi_zip_do_work(req, qp_ctx);
550 if (ret != -EINPROGRESS)
551 hisi_zip_remove_req(qp_ctx, req);
552
553 return ret;
554 }
555
556 static struct acomp_alg hisi_zip_acomp_zlib = {
557 .init = hisi_zip_acomp_init,
558 .exit = hisi_zip_acomp_exit,
559 .compress = hisi_zip_acompress,
560 .decompress = hisi_zip_adecompress,
561 .base = {
562 .cra_name = "zlib-deflate",
563 .cra_driver_name = "hisi-zlib-acomp",
564 .cra_module = THIS_MODULE,
565 .cra_priority = HZIP_ALG_PRIORITY,
566 .cra_ctxsize = sizeof(struct hisi_zip_ctx),
567 }
568 };
569
570 static struct acomp_alg hisi_zip_acomp_gzip = {
571 .init = hisi_zip_acomp_init,
572 .exit = hisi_zip_acomp_exit,
573 .compress = hisi_zip_acompress,
574 .decompress = hisi_zip_adecompress,
575 .base = {
576 .cra_name = "gzip",
577 .cra_driver_name = "hisi-gzip-acomp",
578 .cra_module = THIS_MODULE,
579 .cra_priority = HZIP_ALG_PRIORITY,
580 .cra_ctxsize = sizeof(struct hisi_zip_ctx),
581 }
582 };
583
584 int hisi_zip_register_to_crypto(void)
585 {
586 int ret = 0;
587
588 ret = crypto_register_acomp(&hisi_zip_acomp_zlib);
589 if (ret) {
590 pr_err("Zlib acomp algorithm registration failed\n");
591 return ret;
592 }
593
594 ret = crypto_register_acomp(&hisi_zip_acomp_gzip);
595 if (ret) {
596 pr_err("Gzip acomp algorithm registration failed\n");
597 crypto_unregister_acomp(&hisi_zip_acomp_zlib);
598 }
599
600 return ret;
601 }
602
603 void hisi_zip_unregister_from_crypto(void)
604 {
605 crypto_unregister_acomp(&hisi_zip_acomp_gzip);
606 crypto_unregister_acomp(&hisi_zip_acomp_zlib);
607 }