This source file includes following definitions.
- chacha_doneon
- chacha_neon_stream_xor
- chacha_neon
- xchacha_neon
- chacha_simd_mod_init
- chacha_simd_mod_fini
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 #include <crypto/algapi.h>
23 #include <crypto/chacha.h>
24 #include <crypto/internal/simd.h>
25 #include <crypto/internal/skcipher.h>
26 #include <linux/kernel.h>
27 #include <linux/module.h>
28
29 #include <asm/hwcap.h>
30 #include <asm/neon.h>
31 #include <asm/simd.h>
32
33 asmlinkage void chacha_block_xor_neon(const u32 *state, u8 *dst, const u8 *src,
34 int nrounds);
35 asmlinkage void chacha_4block_xor_neon(const u32 *state, u8 *dst, const u8 *src,
36 int nrounds);
37 asmlinkage void hchacha_block_neon(const u32 *state, u32 *out, int nrounds);
38
39 static void chacha_doneon(u32 *state, u8 *dst, const u8 *src,
40 unsigned int bytes, int nrounds)
41 {
42 u8 buf[CHACHA_BLOCK_SIZE];
43
44 while (bytes >= CHACHA_BLOCK_SIZE * 4) {
45 chacha_4block_xor_neon(state, dst, src, nrounds);
46 bytes -= CHACHA_BLOCK_SIZE * 4;
47 src += CHACHA_BLOCK_SIZE * 4;
48 dst += CHACHA_BLOCK_SIZE * 4;
49 state[12] += 4;
50 }
51 while (bytes >= CHACHA_BLOCK_SIZE) {
52 chacha_block_xor_neon(state, dst, src, nrounds);
53 bytes -= CHACHA_BLOCK_SIZE;
54 src += CHACHA_BLOCK_SIZE;
55 dst += CHACHA_BLOCK_SIZE;
56 state[12]++;
57 }
58 if (bytes) {
59 memcpy(buf, src, bytes);
60 chacha_block_xor_neon(state, buf, buf, nrounds);
61 memcpy(dst, buf, bytes);
62 }
63 }
64
65 static int chacha_neon_stream_xor(struct skcipher_request *req,
66 const struct chacha_ctx *ctx, const u8 *iv)
67 {
68 struct skcipher_walk walk;
69 u32 state[16];
70 int err;
71
72 err = skcipher_walk_virt(&walk, req, false);
73
74 crypto_chacha_init(state, ctx, iv);
75
76 while (walk.nbytes > 0) {
77 unsigned int nbytes = walk.nbytes;
78
79 if (nbytes < walk.total)
80 nbytes = round_down(nbytes, walk.stride);
81
82 kernel_neon_begin();
83 chacha_doneon(state, walk.dst.virt.addr, walk.src.virt.addr,
84 nbytes, ctx->nrounds);
85 kernel_neon_end();
86 err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
87 }
88
89 return err;
90 }
91
92 static int chacha_neon(struct skcipher_request *req)
93 {
94 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
95 struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
96
97 if (req->cryptlen <= CHACHA_BLOCK_SIZE || !crypto_simd_usable())
98 return crypto_chacha_crypt(req);
99
100 return chacha_neon_stream_xor(req, ctx, req->iv);
101 }
102
103 static int xchacha_neon(struct skcipher_request *req)
104 {
105 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
106 struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
107 struct chacha_ctx subctx;
108 u32 state[16];
109 u8 real_iv[16];
110
111 if (req->cryptlen <= CHACHA_BLOCK_SIZE || !crypto_simd_usable())
112 return crypto_xchacha_crypt(req);
113
114 crypto_chacha_init(state, ctx, req->iv);
115
116 kernel_neon_begin();
117 hchacha_block_neon(state, subctx.key, ctx->nrounds);
118 kernel_neon_end();
119 subctx.nrounds = ctx->nrounds;
120
121 memcpy(&real_iv[0], req->iv + 24, 8);
122 memcpy(&real_iv[8], req->iv + 16, 8);
123 return chacha_neon_stream_xor(req, &subctx, real_iv);
124 }
125
126 static struct skcipher_alg algs[] = {
127 {
128 .base.cra_name = "chacha20",
129 .base.cra_driver_name = "chacha20-neon",
130 .base.cra_priority = 300,
131 .base.cra_blocksize = 1,
132 .base.cra_ctxsize = sizeof(struct chacha_ctx),
133 .base.cra_module = THIS_MODULE,
134
135 .min_keysize = CHACHA_KEY_SIZE,
136 .max_keysize = CHACHA_KEY_SIZE,
137 .ivsize = CHACHA_IV_SIZE,
138 .chunksize = CHACHA_BLOCK_SIZE,
139 .walksize = 4 * CHACHA_BLOCK_SIZE,
140 .setkey = crypto_chacha20_setkey,
141 .encrypt = chacha_neon,
142 .decrypt = chacha_neon,
143 }, {
144 .base.cra_name = "xchacha20",
145 .base.cra_driver_name = "xchacha20-neon",
146 .base.cra_priority = 300,
147 .base.cra_blocksize = 1,
148 .base.cra_ctxsize = sizeof(struct chacha_ctx),
149 .base.cra_module = THIS_MODULE,
150
151 .min_keysize = CHACHA_KEY_SIZE,
152 .max_keysize = CHACHA_KEY_SIZE,
153 .ivsize = XCHACHA_IV_SIZE,
154 .chunksize = CHACHA_BLOCK_SIZE,
155 .walksize = 4 * CHACHA_BLOCK_SIZE,
156 .setkey = crypto_chacha20_setkey,
157 .encrypt = xchacha_neon,
158 .decrypt = xchacha_neon,
159 }, {
160 .base.cra_name = "xchacha12",
161 .base.cra_driver_name = "xchacha12-neon",
162 .base.cra_priority = 300,
163 .base.cra_blocksize = 1,
164 .base.cra_ctxsize = sizeof(struct chacha_ctx),
165 .base.cra_module = THIS_MODULE,
166
167 .min_keysize = CHACHA_KEY_SIZE,
168 .max_keysize = CHACHA_KEY_SIZE,
169 .ivsize = XCHACHA_IV_SIZE,
170 .chunksize = CHACHA_BLOCK_SIZE,
171 .walksize = 4 * CHACHA_BLOCK_SIZE,
172 .setkey = crypto_chacha12_setkey,
173 .encrypt = xchacha_neon,
174 .decrypt = xchacha_neon,
175 }
176 };
177
178 static int __init chacha_simd_mod_init(void)
179 {
180 if (!(elf_hwcap & HWCAP_NEON))
181 return -ENODEV;
182
183 return crypto_register_skciphers(algs, ARRAY_SIZE(algs));
184 }
185
186 static void __exit chacha_simd_mod_fini(void)
187 {
188 crypto_unregister_skciphers(algs, ARRAY_SIZE(algs));
189 }
190
191 module_init(chacha_simd_mod_init);
192 module_exit(chacha_simd_mod_fini);
193
194 MODULE_DESCRIPTION("ChaCha and XChaCha stream ciphers (NEON accelerated)");
195 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
196 MODULE_LICENSE("GPL v2");
197 MODULE_ALIAS_CRYPTO("chacha20");
198 MODULE_ALIAS_CRYPTO("chacha20-neon");
199 MODULE_ALIAS_CRYPTO("xchacha20");
200 MODULE_ALIAS_CRYPTO("xchacha20-neon");
201 MODULE_ALIAS_CRYPTO("xchacha12");
202 MODULE_ALIAS_CRYPTO("xchacha12-neon");