1/* 2 * Multi buffer SHA1 algorithm Glue Code 3 * 4 * This file is provided under a dual BSD/GPLv2 license. When using or 5 * redistributing this file, you may do so under either license. 6 * 7 * GPL LICENSE SUMMARY 8 * 9 * Copyright(c) 2014 Intel Corporation. 10 * 11 * This program is free software; you can redistribute it and/or modify 12 * it under the terms of version 2 of the GNU General Public License as 13 * published by the Free Software Foundation. 14 * 15 * This program is distributed in the hope that it will be useful, but 16 * WITHOUT ANY WARRANTY; without even the implied warranty of 17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 18 * General Public License for more details. 19 * 20 * Contact Information: 21 * Tim Chen <tim.c.chen@linux.intel.com> 22 * 23 * BSD LICENSE 24 * 25 * Copyright(c) 2014 Intel Corporation. 26 * 27 * Redistribution and use in source and binary forms, with or without 28 * modification, are permitted provided that the following conditions 29 * are met: 30 * 31 * * Redistributions of source code must retain the above copyright 32 * notice, this list of conditions and the following disclaimer. 33 * * Redistributions in binary form must reproduce the above copyright 34 * notice, this list of conditions and the following disclaimer in 35 * the documentation and/or other materials provided with the 36 * distribution. 37 * * Neither the name of Intel Corporation nor the names of its 38 * contributors may be used to endorse or promote products derived 39 * from this software without specific prior written permission. 40 * 41 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 42 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 43 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 44 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 45 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 46 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 47 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 48 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 49 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 50 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 51 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 52 */ 53 54#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 55 56#include <crypto/internal/hash.h> 57#include <linux/init.h> 58#include <linux/module.h> 59#include <linux/mm.h> 60#include <linux/cryptohash.h> 61#include <linux/types.h> 62#include <linux/list.h> 63#include <crypto/scatterwalk.h> 64#include <crypto/sha.h> 65#include <crypto/mcryptd.h> 66#include <crypto/crypto_wq.h> 67#include <asm/byteorder.h> 68#include <asm/i387.h> 69#include <asm/xcr.h> 70#include <asm/xsave.h> 71#include <linux/hardirq.h> 72#include <asm/fpu-internal.h> 73#include "sha_mb_ctx.h" 74 75#define FLUSH_INTERVAL 1000 /* in usec */ 76 77static struct mcryptd_alg_state sha1_mb_alg_state; 78 79struct sha1_mb_ctx { 80 struct mcryptd_ahash *mcryptd_tfm; 81}; 82 83static inline struct mcryptd_hash_request_ctx *cast_hash_to_mcryptd_ctx(struct sha1_hash_ctx *hash_ctx) 84{ 85 struct shash_desc *desc; 86 87 desc = container_of((void *) hash_ctx, struct shash_desc, __ctx); 88 return container_of(desc, struct mcryptd_hash_request_ctx, desc); 89} 90 91static inline struct ahash_request *cast_mcryptd_ctx_to_req(struct mcryptd_hash_request_ctx *ctx) 92{ 93 return container_of((void *) ctx, struct ahash_request, __ctx); 94} 95 96static void req_ctx_init(struct mcryptd_hash_request_ctx *rctx, 97 struct shash_desc *desc) 98{ 99 rctx->flag = HASH_UPDATE; 100} 101 102static asmlinkage void (*sha1_job_mgr_init)(struct sha1_mb_mgr *state); 103static asmlinkage struct job_sha1* (*sha1_job_mgr_submit)(struct sha1_mb_mgr *state, 104 struct job_sha1 *job); 105static asmlinkage struct job_sha1* (*sha1_job_mgr_flush)(struct sha1_mb_mgr *state); 106static asmlinkage struct job_sha1* (*sha1_job_mgr_get_comp_job)(struct sha1_mb_mgr *state); 107 108inline void sha1_init_digest(uint32_t *digest) 109{ 110 static const uint32_t initial_digest[SHA1_DIGEST_LENGTH] = {SHA1_H0, 111 SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 }; 112 memcpy(digest, initial_digest, sizeof(initial_digest)); 113} 114 115inline uint32_t sha1_pad(uint8_t padblock[SHA1_BLOCK_SIZE * 2], 116 uint32_t total_len) 117{ 118 uint32_t i = total_len & (SHA1_BLOCK_SIZE - 1); 119 120 memset(&padblock[i], 0, SHA1_BLOCK_SIZE); 121 padblock[i] = 0x80; 122 123 i += ((SHA1_BLOCK_SIZE - 1) & 124 (0 - (total_len + SHA1_PADLENGTHFIELD_SIZE + 1))) 125 + 1 + SHA1_PADLENGTHFIELD_SIZE; 126 127#if SHA1_PADLENGTHFIELD_SIZE == 16 128 *((uint64_t *) &padblock[i - 16]) = 0; 129#endif 130 131 *((uint64_t *) &padblock[i - 8]) = cpu_to_be64(total_len << 3); 132 133 /* Number of extra blocks to hash */ 134 return i >> SHA1_LOG2_BLOCK_SIZE; 135} 136 137static struct sha1_hash_ctx *sha1_ctx_mgr_resubmit(struct sha1_ctx_mgr *mgr, struct sha1_hash_ctx *ctx) 138{ 139 while (ctx) { 140 if (ctx->status & HASH_CTX_STS_COMPLETE) { 141 /* Clear PROCESSING bit */ 142 ctx->status = HASH_CTX_STS_COMPLETE; 143 return ctx; 144 } 145 146 /* 147 * If the extra blocks are empty, begin hashing what remains 148 * in the user's buffer. 149 */ 150 if (ctx->partial_block_buffer_length == 0 && 151 ctx->incoming_buffer_length) { 152 153 const void *buffer = ctx->incoming_buffer; 154 uint32_t len = ctx->incoming_buffer_length; 155 uint32_t copy_len; 156 157 /* 158 * Only entire blocks can be hashed. 159 * Copy remainder to extra blocks buffer. 160 */ 161 copy_len = len & (SHA1_BLOCK_SIZE-1); 162 163 if (copy_len) { 164 len -= copy_len; 165 memcpy(ctx->partial_block_buffer, 166 ((const char *) buffer + len), 167 copy_len); 168 ctx->partial_block_buffer_length = copy_len; 169 } 170 171 ctx->incoming_buffer_length = 0; 172 173 /* len should be a multiple of the block size now */ 174 assert((len % SHA1_BLOCK_SIZE) == 0); 175 176 /* Set len to the number of blocks to be hashed */ 177 len >>= SHA1_LOG2_BLOCK_SIZE; 178 179 if (len) { 180 181 ctx->job.buffer = (uint8_t *) buffer; 182 ctx->job.len = len; 183 ctx = (struct sha1_hash_ctx *) sha1_job_mgr_submit(&mgr->mgr, 184 &ctx->job); 185 continue; 186 } 187 } 188 189 /* 190 * If the extra blocks are not empty, then we are 191 * either on the last block(s) or we need more 192 * user input before continuing. 193 */ 194 if (ctx->status & HASH_CTX_STS_LAST) { 195 196 uint8_t *buf = ctx->partial_block_buffer; 197 uint32_t n_extra_blocks = sha1_pad(buf, ctx->total_length); 198 199 ctx->status = (HASH_CTX_STS_PROCESSING | 200 HASH_CTX_STS_COMPLETE); 201 ctx->job.buffer = buf; 202 ctx->job.len = (uint32_t) n_extra_blocks; 203 ctx = (struct sha1_hash_ctx *) sha1_job_mgr_submit(&mgr->mgr, &ctx->job); 204 continue; 205 } 206 207 ctx->status = HASH_CTX_STS_IDLE; 208 return ctx; 209 } 210 211 return NULL; 212} 213 214static struct sha1_hash_ctx *sha1_ctx_mgr_get_comp_ctx(struct sha1_ctx_mgr *mgr) 215{ 216 /* 217 * If get_comp_job returns NULL, there are no jobs complete. 218 * If get_comp_job returns a job, verify that it is safe to return to the user. 219 * If it is not ready, resubmit the job to finish processing. 220 * If sha1_ctx_mgr_resubmit returned a job, it is ready to be returned. 221 * Otherwise, all jobs currently being managed by the hash_ctx_mgr still need processing. 222 */ 223 struct sha1_hash_ctx *ctx; 224 225 ctx = (struct sha1_hash_ctx *) sha1_job_mgr_get_comp_job(&mgr->mgr); 226 return sha1_ctx_mgr_resubmit(mgr, ctx); 227} 228 229static void sha1_ctx_mgr_init(struct sha1_ctx_mgr *mgr) 230{ 231 sha1_job_mgr_init(&mgr->mgr); 232} 233 234static struct sha1_hash_ctx *sha1_ctx_mgr_submit(struct sha1_ctx_mgr *mgr, 235 struct sha1_hash_ctx *ctx, 236 const void *buffer, 237 uint32_t len, 238 int flags) 239{ 240 if (flags & (~HASH_ENTIRE)) { 241 /* User should not pass anything other than FIRST, UPDATE, or LAST */ 242 ctx->error = HASH_CTX_ERROR_INVALID_FLAGS; 243 return ctx; 244 } 245 246 if (ctx->status & HASH_CTX_STS_PROCESSING) { 247 /* Cannot submit to a currently processing job. */ 248 ctx->error = HASH_CTX_ERROR_ALREADY_PROCESSING; 249 return ctx; 250 } 251 252 if ((ctx->status & HASH_CTX_STS_COMPLETE) && !(flags & HASH_FIRST)) { 253 /* Cannot update a finished job. */ 254 ctx->error = HASH_CTX_ERROR_ALREADY_COMPLETED; 255 return ctx; 256 } 257 258 259 if (flags & HASH_FIRST) { 260 /* Init digest */ 261 sha1_init_digest(ctx->job.result_digest); 262 263 /* Reset byte counter */ 264 ctx->total_length = 0; 265 266 /* Clear extra blocks */ 267 ctx->partial_block_buffer_length = 0; 268 } 269 270 /* If we made it here, there were no errors during this call to submit */ 271 ctx->error = HASH_CTX_ERROR_NONE; 272 273 /* Store buffer ptr info from user */ 274 ctx->incoming_buffer = buffer; 275 ctx->incoming_buffer_length = len; 276 277 /* Store the user's request flags and mark this ctx as currently being processed. */ 278 ctx->status = (flags & HASH_LAST) ? 279 (HASH_CTX_STS_PROCESSING | HASH_CTX_STS_LAST) : 280 HASH_CTX_STS_PROCESSING; 281 282 /* Advance byte counter */ 283 ctx->total_length += len; 284 285 /* 286 * If there is anything currently buffered in the extra blocks, 287 * append to it until it contains a whole block. 288 * Or if the user's buffer contains less than a whole block, 289 * append as much as possible to the extra block. 290 */ 291 if ((ctx->partial_block_buffer_length) | (len < SHA1_BLOCK_SIZE)) { 292 /* Compute how many bytes to copy from user buffer into extra block */ 293 uint32_t copy_len = SHA1_BLOCK_SIZE - ctx->partial_block_buffer_length; 294 if (len < copy_len) 295 copy_len = len; 296 297 if (copy_len) { 298 /* Copy and update relevant pointers and counters */ 299 memcpy(&ctx->partial_block_buffer[ctx->partial_block_buffer_length], 300 buffer, copy_len); 301 302 ctx->partial_block_buffer_length += copy_len; 303 ctx->incoming_buffer = (const void *)((const char *)buffer + copy_len); 304 ctx->incoming_buffer_length = len - copy_len; 305 } 306 307 /* The extra block should never contain more than 1 block here */ 308 assert(ctx->partial_block_buffer_length <= SHA1_BLOCK_SIZE); 309 310 /* If the extra block buffer contains exactly 1 block, it can be hashed. */ 311 if (ctx->partial_block_buffer_length >= SHA1_BLOCK_SIZE) { 312 ctx->partial_block_buffer_length = 0; 313 314 ctx->job.buffer = ctx->partial_block_buffer; 315 ctx->job.len = 1; 316 ctx = (struct sha1_hash_ctx *) sha1_job_mgr_submit(&mgr->mgr, &ctx->job); 317 } 318 } 319 320 return sha1_ctx_mgr_resubmit(mgr, ctx); 321} 322 323static struct sha1_hash_ctx *sha1_ctx_mgr_flush(struct sha1_ctx_mgr *mgr) 324{ 325 struct sha1_hash_ctx *ctx; 326 327 while (1) { 328 ctx = (struct sha1_hash_ctx *) sha1_job_mgr_flush(&mgr->mgr); 329 330 /* If flush returned 0, there are no more jobs in flight. */ 331 if (!ctx) 332 return NULL; 333 334 /* 335 * If flush returned a job, resubmit the job to finish processing. 336 */ 337 ctx = sha1_ctx_mgr_resubmit(mgr, ctx); 338 339 /* 340 * If sha1_ctx_mgr_resubmit returned a job, it is ready to be returned. 341 * Otherwise, all jobs currently being managed by the sha1_ctx_mgr 342 * still need processing. Loop. 343 */ 344 if (ctx) 345 return ctx; 346 } 347} 348 349static int sha1_mb_init(struct shash_desc *desc) 350{ 351 struct sha1_hash_ctx *sctx = shash_desc_ctx(desc); 352 353 hash_ctx_init(sctx); 354 sctx->job.result_digest[0] = SHA1_H0; 355 sctx->job.result_digest[1] = SHA1_H1; 356 sctx->job.result_digest[2] = SHA1_H2; 357 sctx->job.result_digest[3] = SHA1_H3; 358 sctx->job.result_digest[4] = SHA1_H4; 359 sctx->total_length = 0; 360 sctx->partial_block_buffer_length = 0; 361 sctx->status = HASH_CTX_STS_IDLE; 362 363 return 0; 364} 365 366static int sha1_mb_set_results(struct mcryptd_hash_request_ctx *rctx) 367{ 368 int i; 369 struct sha1_hash_ctx *sctx = shash_desc_ctx(&rctx->desc); 370 __be32 *dst = (__be32 *) rctx->out; 371 372 for (i = 0; i < 5; ++i) 373 dst[i] = cpu_to_be32(sctx->job.result_digest[i]); 374 375 return 0; 376} 377 378static int sha_finish_walk(struct mcryptd_hash_request_ctx **ret_rctx, 379 struct mcryptd_alg_cstate *cstate, bool flush) 380{ 381 int flag = HASH_UPDATE; 382 int nbytes, err = 0; 383 struct mcryptd_hash_request_ctx *rctx = *ret_rctx; 384 struct sha1_hash_ctx *sha_ctx; 385 386 /* more work ? */ 387 while (!(rctx->flag & HASH_DONE)) { 388 nbytes = crypto_ahash_walk_done(&rctx->walk, 0); 389 if (nbytes < 0) { 390 err = nbytes; 391 goto out; 392 } 393 /* check if the walk is done */ 394 if (crypto_ahash_walk_last(&rctx->walk)) { 395 rctx->flag |= HASH_DONE; 396 if (rctx->flag & HASH_FINAL) 397 flag |= HASH_LAST; 398 399 } 400 sha_ctx = (struct sha1_hash_ctx *) shash_desc_ctx(&rctx->desc); 401 kernel_fpu_begin(); 402 sha_ctx = sha1_ctx_mgr_submit(cstate->mgr, sha_ctx, rctx->walk.data, nbytes, flag); 403 if (!sha_ctx) { 404 if (flush) 405 sha_ctx = sha1_ctx_mgr_flush(cstate->mgr); 406 } 407 kernel_fpu_end(); 408 if (sha_ctx) 409 rctx = cast_hash_to_mcryptd_ctx(sha_ctx); 410 else { 411 rctx = NULL; 412 goto out; 413 } 414 } 415 416 /* copy the results */ 417 if (rctx->flag & HASH_FINAL) 418 sha1_mb_set_results(rctx); 419 420out: 421 *ret_rctx = rctx; 422 return err; 423} 424 425static int sha_complete_job(struct mcryptd_hash_request_ctx *rctx, 426 struct mcryptd_alg_cstate *cstate, 427 int err) 428{ 429 struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx); 430 struct sha1_hash_ctx *sha_ctx; 431 struct mcryptd_hash_request_ctx *req_ctx; 432 int ret; 433 434 /* remove from work list */ 435 spin_lock(&cstate->work_lock); 436 list_del(&rctx->waiter); 437 spin_unlock(&cstate->work_lock); 438 439 if (irqs_disabled()) 440 rctx->complete(&req->base, err); 441 else { 442 local_bh_disable(); 443 rctx->complete(&req->base, err); 444 local_bh_enable(); 445 } 446 447 /* check to see if there are other jobs that are done */ 448 sha_ctx = sha1_ctx_mgr_get_comp_ctx(cstate->mgr); 449 while (sha_ctx) { 450 req_ctx = cast_hash_to_mcryptd_ctx(sha_ctx); 451 ret = sha_finish_walk(&req_ctx, cstate, false); 452 if (req_ctx) { 453 spin_lock(&cstate->work_lock); 454 list_del(&req_ctx->waiter); 455 spin_unlock(&cstate->work_lock); 456 457 req = cast_mcryptd_ctx_to_req(req_ctx); 458 if (irqs_disabled()) 459 req_ctx->complete(&req->base, ret); 460 else { 461 local_bh_disable(); 462 req_ctx->complete(&req->base, ret); 463 local_bh_enable(); 464 } 465 } 466 sha_ctx = sha1_ctx_mgr_get_comp_ctx(cstate->mgr); 467 } 468 469 return 0; 470} 471 472static void sha1_mb_add_list(struct mcryptd_hash_request_ctx *rctx, 473 struct mcryptd_alg_cstate *cstate) 474{ 475 unsigned long next_flush; 476 unsigned long delay = usecs_to_jiffies(FLUSH_INTERVAL); 477 478 /* initialize tag */ 479 rctx->tag.arrival = jiffies; /* tag the arrival time */ 480 rctx->tag.seq_num = cstate->next_seq_num++; 481 next_flush = rctx->tag.arrival + delay; 482 rctx->tag.expire = next_flush; 483 484 spin_lock(&cstate->work_lock); 485 list_add_tail(&rctx->waiter, &cstate->work_list); 486 spin_unlock(&cstate->work_lock); 487 488 mcryptd_arm_flusher(cstate, delay); 489} 490 491static int sha1_mb_update(struct shash_desc *desc, const u8 *data, 492 unsigned int len) 493{ 494 struct mcryptd_hash_request_ctx *rctx = 495 container_of(desc, struct mcryptd_hash_request_ctx, desc); 496 struct mcryptd_alg_cstate *cstate = 497 this_cpu_ptr(sha1_mb_alg_state.alg_cstate); 498 499 struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx); 500 struct sha1_hash_ctx *sha_ctx; 501 int ret = 0, nbytes; 502 503 504 /* sanity check */ 505 if (rctx->tag.cpu != smp_processor_id()) { 506 pr_err("mcryptd error: cpu clash\n"); 507 goto done; 508 } 509 510 /* need to init context */ 511 req_ctx_init(rctx, desc); 512 513 nbytes = crypto_ahash_walk_first(req, &rctx->walk); 514 515 if (nbytes < 0) { 516 ret = nbytes; 517 goto done; 518 } 519 520 if (crypto_ahash_walk_last(&rctx->walk)) 521 rctx->flag |= HASH_DONE; 522 523 /* submit */ 524 sha_ctx = (struct sha1_hash_ctx *) shash_desc_ctx(desc); 525 sha1_mb_add_list(rctx, cstate); 526 kernel_fpu_begin(); 527 sha_ctx = sha1_ctx_mgr_submit(cstate->mgr, sha_ctx, rctx->walk.data, nbytes, HASH_UPDATE); 528 kernel_fpu_end(); 529 530 /* check if anything is returned */ 531 if (!sha_ctx) 532 return -EINPROGRESS; 533 534 if (sha_ctx->error) { 535 ret = sha_ctx->error; 536 rctx = cast_hash_to_mcryptd_ctx(sha_ctx); 537 goto done; 538 } 539 540 rctx = cast_hash_to_mcryptd_ctx(sha_ctx); 541 ret = sha_finish_walk(&rctx, cstate, false); 542 543 if (!rctx) 544 return -EINPROGRESS; 545done: 546 sha_complete_job(rctx, cstate, ret); 547 return ret; 548} 549 550static int sha1_mb_finup(struct shash_desc *desc, const u8 *data, 551 unsigned int len, u8 *out) 552{ 553 struct mcryptd_hash_request_ctx *rctx = 554 container_of(desc, struct mcryptd_hash_request_ctx, desc); 555 struct mcryptd_alg_cstate *cstate = 556 this_cpu_ptr(sha1_mb_alg_state.alg_cstate); 557 558 struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx); 559 struct sha1_hash_ctx *sha_ctx; 560 int ret = 0, flag = HASH_UPDATE, nbytes; 561 562 /* sanity check */ 563 if (rctx->tag.cpu != smp_processor_id()) { 564 pr_err("mcryptd error: cpu clash\n"); 565 goto done; 566 } 567 568 /* need to init context */ 569 req_ctx_init(rctx, desc); 570 571 nbytes = crypto_ahash_walk_first(req, &rctx->walk); 572 573 if (nbytes < 0) { 574 ret = nbytes; 575 goto done; 576 } 577 578 if (crypto_ahash_walk_last(&rctx->walk)) { 579 rctx->flag |= HASH_DONE; 580 flag = HASH_LAST; 581 } 582 rctx->out = out; 583 584 /* submit */ 585 rctx->flag |= HASH_FINAL; 586 sha_ctx = (struct sha1_hash_ctx *) shash_desc_ctx(desc); 587 sha1_mb_add_list(rctx, cstate); 588 589 kernel_fpu_begin(); 590 sha_ctx = sha1_ctx_mgr_submit(cstate->mgr, sha_ctx, rctx->walk.data, nbytes, flag); 591 kernel_fpu_end(); 592 593 /* check if anything is returned */ 594 if (!sha_ctx) 595 return -EINPROGRESS; 596 597 if (sha_ctx->error) { 598 ret = sha_ctx->error; 599 goto done; 600 } 601 602 rctx = cast_hash_to_mcryptd_ctx(sha_ctx); 603 ret = sha_finish_walk(&rctx, cstate, false); 604 if (!rctx) 605 return -EINPROGRESS; 606done: 607 sha_complete_job(rctx, cstate, ret); 608 return ret; 609} 610 611static int sha1_mb_final(struct shash_desc *desc, u8 *out) 612{ 613 struct mcryptd_hash_request_ctx *rctx = 614 container_of(desc, struct mcryptd_hash_request_ctx, desc); 615 struct mcryptd_alg_cstate *cstate = 616 this_cpu_ptr(sha1_mb_alg_state.alg_cstate); 617 618 struct sha1_hash_ctx *sha_ctx; 619 int ret = 0; 620 u8 data; 621 622 /* sanity check */ 623 if (rctx->tag.cpu != smp_processor_id()) { 624 pr_err("mcryptd error: cpu clash\n"); 625 goto done; 626 } 627 628 /* need to init context */ 629 req_ctx_init(rctx, desc); 630 631 rctx->out = out; 632 rctx->flag |= HASH_DONE | HASH_FINAL; 633 634 sha_ctx = (struct sha1_hash_ctx *) shash_desc_ctx(desc); 635 /* flag HASH_FINAL and 0 data size */ 636 sha1_mb_add_list(rctx, cstate); 637 kernel_fpu_begin(); 638 sha_ctx = sha1_ctx_mgr_submit(cstate->mgr, sha_ctx, &data, 0, HASH_LAST); 639 kernel_fpu_end(); 640 641 /* check if anything is returned */ 642 if (!sha_ctx) 643 return -EINPROGRESS; 644 645 if (sha_ctx->error) { 646 ret = sha_ctx->error; 647 rctx = cast_hash_to_mcryptd_ctx(sha_ctx); 648 goto done; 649 } 650 651 rctx = cast_hash_to_mcryptd_ctx(sha_ctx); 652 ret = sha_finish_walk(&rctx, cstate, false); 653 if (!rctx) 654 return -EINPROGRESS; 655done: 656 sha_complete_job(rctx, cstate, ret); 657 return ret; 658} 659 660static int sha1_mb_export(struct shash_desc *desc, void *out) 661{ 662 struct sha1_hash_ctx *sctx = shash_desc_ctx(desc); 663 664 memcpy(out, sctx, sizeof(*sctx)); 665 666 return 0; 667} 668 669static int sha1_mb_import(struct shash_desc *desc, const void *in) 670{ 671 struct sha1_hash_ctx *sctx = shash_desc_ctx(desc); 672 673 memcpy(sctx, in, sizeof(*sctx)); 674 675 return 0; 676} 677 678 679static struct shash_alg sha1_mb_shash_alg = { 680 .digestsize = SHA1_DIGEST_SIZE, 681 .init = sha1_mb_init, 682 .update = sha1_mb_update, 683 .final = sha1_mb_final, 684 .finup = sha1_mb_finup, 685 .export = sha1_mb_export, 686 .import = sha1_mb_import, 687 .descsize = sizeof(struct sha1_hash_ctx), 688 .statesize = sizeof(struct sha1_hash_ctx), 689 .base = { 690 .cra_name = "__sha1-mb", 691 .cra_driver_name = "__intel_sha1-mb", 692 .cra_priority = 100, 693 /* 694 * use ASYNC flag as some buffers in multi-buffer 695 * algo may not have completed before hashing thread sleep 696 */ 697 .cra_flags = CRYPTO_ALG_TYPE_SHASH | CRYPTO_ALG_ASYNC | 698 CRYPTO_ALG_INTERNAL, 699 .cra_blocksize = SHA1_BLOCK_SIZE, 700 .cra_module = THIS_MODULE, 701 .cra_list = LIST_HEAD_INIT(sha1_mb_shash_alg.base.cra_list), 702 } 703}; 704 705static int sha1_mb_async_init(struct ahash_request *req) 706{ 707 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 708 struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm); 709 struct ahash_request *mcryptd_req = ahash_request_ctx(req); 710 struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; 711 712 memcpy(mcryptd_req, req, sizeof(*req)); 713 ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); 714 return crypto_ahash_init(mcryptd_req); 715} 716 717static int sha1_mb_async_update(struct ahash_request *req) 718{ 719 struct ahash_request *mcryptd_req = ahash_request_ctx(req); 720 721 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 722 struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm); 723 struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; 724 725 memcpy(mcryptd_req, req, sizeof(*req)); 726 ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); 727 return crypto_ahash_update(mcryptd_req); 728} 729 730static int sha1_mb_async_finup(struct ahash_request *req) 731{ 732 struct ahash_request *mcryptd_req = ahash_request_ctx(req); 733 734 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 735 struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm); 736 struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; 737 738 memcpy(mcryptd_req, req, sizeof(*req)); 739 ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); 740 return crypto_ahash_finup(mcryptd_req); 741} 742 743static int sha1_mb_async_final(struct ahash_request *req) 744{ 745 struct ahash_request *mcryptd_req = ahash_request_ctx(req); 746 747 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 748 struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm); 749 struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; 750 751 memcpy(mcryptd_req, req, sizeof(*req)); 752 ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); 753 return crypto_ahash_final(mcryptd_req); 754} 755 756static int sha1_mb_async_digest(struct ahash_request *req) 757{ 758 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 759 struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm); 760 struct ahash_request *mcryptd_req = ahash_request_ctx(req); 761 struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; 762 763 memcpy(mcryptd_req, req, sizeof(*req)); 764 ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); 765 return crypto_ahash_digest(mcryptd_req); 766} 767 768static int sha1_mb_async_init_tfm(struct crypto_tfm *tfm) 769{ 770 struct mcryptd_ahash *mcryptd_tfm; 771 struct sha1_mb_ctx *ctx = crypto_tfm_ctx(tfm); 772 struct mcryptd_hash_ctx *mctx; 773 774 mcryptd_tfm = mcryptd_alloc_ahash("__intel_sha1-mb", 775 CRYPTO_ALG_INTERNAL, 776 CRYPTO_ALG_INTERNAL); 777 if (IS_ERR(mcryptd_tfm)) 778 return PTR_ERR(mcryptd_tfm); 779 mctx = crypto_ahash_ctx(&mcryptd_tfm->base); 780 mctx->alg_state = &sha1_mb_alg_state; 781 ctx->mcryptd_tfm = mcryptd_tfm; 782 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 783 sizeof(struct ahash_request) + 784 crypto_ahash_reqsize(&mcryptd_tfm->base)); 785 786 return 0; 787} 788 789static void sha1_mb_async_exit_tfm(struct crypto_tfm *tfm) 790{ 791 struct sha1_mb_ctx *ctx = crypto_tfm_ctx(tfm); 792 793 mcryptd_free_ahash(ctx->mcryptd_tfm); 794} 795 796static struct ahash_alg sha1_mb_async_alg = { 797 .init = sha1_mb_async_init, 798 .update = sha1_mb_async_update, 799 .final = sha1_mb_async_final, 800 .finup = sha1_mb_async_finup, 801 .digest = sha1_mb_async_digest, 802 .halg = { 803 .digestsize = SHA1_DIGEST_SIZE, 804 .base = { 805 .cra_name = "sha1", 806 .cra_driver_name = "sha1_mb", 807 .cra_priority = 200, 808 .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC, 809 .cra_blocksize = SHA1_BLOCK_SIZE, 810 .cra_type = &crypto_ahash_type, 811 .cra_module = THIS_MODULE, 812 .cra_list = LIST_HEAD_INIT(sha1_mb_async_alg.halg.base.cra_list), 813 .cra_init = sha1_mb_async_init_tfm, 814 .cra_exit = sha1_mb_async_exit_tfm, 815 .cra_ctxsize = sizeof(struct sha1_mb_ctx), 816 .cra_alignmask = 0, 817 }, 818 }, 819}; 820 821static unsigned long sha1_mb_flusher(struct mcryptd_alg_cstate *cstate) 822{ 823 struct mcryptd_hash_request_ctx *rctx; 824 unsigned long cur_time; 825 unsigned long next_flush = 0; 826 struct sha1_hash_ctx *sha_ctx; 827 828 829 cur_time = jiffies; 830 831 while (!list_empty(&cstate->work_list)) { 832 rctx = list_entry(cstate->work_list.next, 833 struct mcryptd_hash_request_ctx, waiter); 834 if (time_before(cur_time, rctx->tag.expire)) 835 break; 836 kernel_fpu_begin(); 837 sha_ctx = (struct sha1_hash_ctx *) sha1_ctx_mgr_flush(cstate->mgr); 838 kernel_fpu_end(); 839 if (!sha_ctx) { 840 pr_err("sha1_mb error: nothing got flushed for non-empty list\n"); 841 break; 842 } 843 rctx = cast_hash_to_mcryptd_ctx(sha_ctx); 844 sha_finish_walk(&rctx, cstate, true); 845 sha_complete_job(rctx, cstate, 0); 846 } 847 848 if (!list_empty(&cstate->work_list)) { 849 rctx = list_entry(cstate->work_list.next, 850 struct mcryptd_hash_request_ctx, waiter); 851 /* get the hash context and then flush time */ 852 next_flush = rctx->tag.expire; 853 mcryptd_arm_flusher(cstate, get_delay(next_flush)); 854 } 855 return next_flush; 856} 857 858static int __init sha1_mb_mod_init(void) 859{ 860 861 int cpu; 862 int err; 863 struct mcryptd_alg_cstate *cpu_state; 864 865 /* check for dependent cpu features */ 866 if (!boot_cpu_has(X86_FEATURE_AVX2) || 867 !boot_cpu_has(X86_FEATURE_BMI2)) 868 return -ENODEV; 869 870 /* initialize multibuffer structures */ 871 sha1_mb_alg_state.alg_cstate = alloc_percpu(struct mcryptd_alg_cstate); 872 873 sha1_job_mgr_init = sha1_mb_mgr_init_avx2; 874 sha1_job_mgr_submit = sha1_mb_mgr_submit_avx2; 875 sha1_job_mgr_flush = sha1_mb_mgr_flush_avx2; 876 sha1_job_mgr_get_comp_job = sha1_mb_mgr_get_comp_job_avx2; 877 878 if (!sha1_mb_alg_state.alg_cstate) 879 return -ENOMEM; 880 for_each_possible_cpu(cpu) { 881 cpu_state = per_cpu_ptr(sha1_mb_alg_state.alg_cstate, cpu); 882 cpu_state->next_flush = 0; 883 cpu_state->next_seq_num = 0; 884 cpu_state->flusher_engaged = false; 885 INIT_DELAYED_WORK(&cpu_state->flush, mcryptd_flusher); 886 cpu_state->cpu = cpu; 887 cpu_state->alg_state = &sha1_mb_alg_state; 888 cpu_state->mgr = (struct sha1_ctx_mgr *) kzalloc(sizeof(struct sha1_ctx_mgr), GFP_KERNEL); 889 if (!cpu_state->mgr) 890 goto err2; 891 sha1_ctx_mgr_init(cpu_state->mgr); 892 INIT_LIST_HEAD(&cpu_state->work_list); 893 spin_lock_init(&cpu_state->work_lock); 894 } 895 sha1_mb_alg_state.flusher = &sha1_mb_flusher; 896 897 err = crypto_register_shash(&sha1_mb_shash_alg); 898 if (err) 899 goto err2; 900 err = crypto_register_ahash(&sha1_mb_async_alg); 901 if (err) 902 goto err1; 903 904 905 return 0; 906err1: 907 crypto_unregister_shash(&sha1_mb_shash_alg); 908err2: 909 for_each_possible_cpu(cpu) { 910 cpu_state = per_cpu_ptr(sha1_mb_alg_state.alg_cstate, cpu); 911 kfree(cpu_state->mgr); 912 } 913 free_percpu(sha1_mb_alg_state.alg_cstate); 914 return -ENODEV; 915} 916 917static void __exit sha1_mb_mod_fini(void) 918{ 919 int cpu; 920 struct mcryptd_alg_cstate *cpu_state; 921 922 crypto_unregister_ahash(&sha1_mb_async_alg); 923 crypto_unregister_shash(&sha1_mb_shash_alg); 924 for_each_possible_cpu(cpu) { 925 cpu_state = per_cpu_ptr(sha1_mb_alg_state.alg_cstate, cpu); 926 kfree(cpu_state->mgr); 927 } 928 free_percpu(sha1_mb_alg_state.alg_cstate); 929} 930 931module_init(sha1_mb_mod_init); 932module_exit(sha1_mb_mod_fini); 933 934MODULE_LICENSE("GPL"); 935MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm, multi buffer accelerated"); 936 937MODULE_ALIAS_CRYPTO("sha1"); 938