This source file includes following definitions.
- crc32_cra_init
- crc32c_cra_init
- crc32_setkey
- crc32_init
- crc32_update
- crc32c_update
- crc32_final
- crc32c_final
- crc32_pmull_update
- crc32c_pmull_update
- crc32_pmull_mod_init
- crc32_pmull_mod_exit
1
2
3
4
5
6
7
8 #include <linux/cpufeature.h>
9 #include <linux/crc32.h>
10 #include <linux/init.h>
11 #include <linux/kernel.h>
12 #include <linux/module.h>
13 #include <linux/string.h>
14
15 #include <crypto/internal/hash.h>
16 #include <crypto/internal/simd.h>
17
18 #include <asm/hwcap.h>
19 #include <asm/neon.h>
20 #include <asm/simd.h>
21 #include <asm/unaligned.h>
22
23 #define PMULL_MIN_LEN 64L
24
25 #define SCALE_F 16L
26
27 asmlinkage u32 crc32_pmull_le(const u8 buf[], u32 len, u32 init_crc);
28 asmlinkage u32 crc32_armv8_le(u32 init_crc, const u8 buf[], u32 len);
29
30 asmlinkage u32 crc32c_pmull_le(const u8 buf[], u32 len, u32 init_crc);
31 asmlinkage u32 crc32c_armv8_le(u32 init_crc, const u8 buf[], u32 len);
32
33 static u32 (*fallback_crc32)(u32 init_crc, const u8 buf[], u32 len);
34 static u32 (*fallback_crc32c)(u32 init_crc, const u8 buf[], u32 len);
35
36 static int crc32_cra_init(struct crypto_tfm *tfm)
37 {
38 u32 *key = crypto_tfm_ctx(tfm);
39
40 *key = 0;
41 return 0;
42 }
43
44 static int crc32c_cra_init(struct crypto_tfm *tfm)
45 {
46 u32 *key = crypto_tfm_ctx(tfm);
47
48 *key = ~0;
49 return 0;
50 }
51
52 static int crc32_setkey(struct crypto_shash *hash, const u8 *key,
53 unsigned int keylen)
54 {
55 u32 *mctx = crypto_shash_ctx(hash);
56
57 if (keylen != sizeof(u32)) {
58 crypto_shash_set_flags(hash, CRYPTO_TFM_RES_BAD_KEY_LEN);
59 return -EINVAL;
60 }
61 *mctx = le32_to_cpup((__le32 *)key);
62 return 0;
63 }
64
65 static int crc32_init(struct shash_desc *desc)
66 {
67 u32 *mctx = crypto_shash_ctx(desc->tfm);
68 u32 *crc = shash_desc_ctx(desc);
69
70 *crc = *mctx;
71 return 0;
72 }
73
74 static int crc32_update(struct shash_desc *desc, const u8 *data,
75 unsigned int length)
76 {
77 u32 *crc = shash_desc_ctx(desc);
78
79 *crc = crc32_armv8_le(*crc, data, length);
80 return 0;
81 }
82
83 static int crc32c_update(struct shash_desc *desc, const u8 *data,
84 unsigned int length)
85 {
86 u32 *crc = shash_desc_ctx(desc);
87
88 *crc = crc32c_armv8_le(*crc, data, length);
89 return 0;
90 }
91
92 static int crc32_final(struct shash_desc *desc, u8 *out)
93 {
94 u32 *crc = shash_desc_ctx(desc);
95
96 put_unaligned_le32(*crc, out);
97 return 0;
98 }
99
100 static int crc32c_final(struct shash_desc *desc, u8 *out)
101 {
102 u32 *crc = shash_desc_ctx(desc);
103
104 put_unaligned_le32(~*crc, out);
105 return 0;
106 }
107
108 static int crc32_pmull_update(struct shash_desc *desc, const u8 *data,
109 unsigned int length)
110 {
111 u32 *crc = shash_desc_ctx(desc);
112 unsigned int l;
113
114 if (crypto_simd_usable()) {
115 if ((u32)data % SCALE_F) {
116 l = min_t(u32, length, SCALE_F - ((u32)data % SCALE_F));
117
118 *crc = fallback_crc32(*crc, data, l);
119
120 data += l;
121 length -= l;
122 }
123
124 if (length >= PMULL_MIN_LEN) {
125 l = round_down(length, SCALE_F);
126
127 kernel_neon_begin();
128 *crc = crc32_pmull_le(data, l, *crc);
129 kernel_neon_end();
130
131 data += l;
132 length -= l;
133 }
134 }
135
136 if (length > 0)
137 *crc = fallback_crc32(*crc, data, length);
138
139 return 0;
140 }
141
142 static int crc32c_pmull_update(struct shash_desc *desc, const u8 *data,
143 unsigned int length)
144 {
145 u32 *crc = shash_desc_ctx(desc);
146 unsigned int l;
147
148 if (crypto_simd_usable()) {
149 if ((u32)data % SCALE_F) {
150 l = min_t(u32, length, SCALE_F - ((u32)data % SCALE_F));
151
152 *crc = fallback_crc32c(*crc, data, l);
153
154 data += l;
155 length -= l;
156 }
157
158 if (length >= PMULL_MIN_LEN) {
159 l = round_down(length, SCALE_F);
160
161 kernel_neon_begin();
162 *crc = crc32c_pmull_le(data, l, *crc);
163 kernel_neon_end();
164
165 data += l;
166 length -= l;
167 }
168 }
169
170 if (length > 0)
171 *crc = fallback_crc32c(*crc, data, length);
172
173 return 0;
174 }
175
176 static struct shash_alg crc32_pmull_algs[] = { {
177 .setkey = crc32_setkey,
178 .init = crc32_init,
179 .update = crc32_update,
180 .final = crc32_final,
181 .descsize = sizeof(u32),
182 .digestsize = sizeof(u32),
183
184 .base.cra_ctxsize = sizeof(u32),
185 .base.cra_init = crc32_cra_init,
186 .base.cra_name = "crc32",
187 .base.cra_driver_name = "crc32-arm-ce",
188 .base.cra_priority = 200,
189 .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
190 .base.cra_blocksize = 1,
191 .base.cra_module = THIS_MODULE,
192 }, {
193 .setkey = crc32_setkey,
194 .init = crc32_init,
195 .update = crc32c_update,
196 .final = crc32c_final,
197 .descsize = sizeof(u32),
198 .digestsize = sizeof(u32),
199
200 .base.cra_ctxsize = sizeof(u32),
201 .base.cra_init = crc32c_cra_init,
202 .base.cra_name = "crc32c",
203 .base.cra_driver_name = "crc32c-arm-ce",
204 .base.cra_priority = 200,
205 .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
206 .base.cra_blocksize = 1,
207 .base.cra_module = THIS_MODULE,
208 } };
209
210 static int __init crc32_pmull_mod_init(void)
211 {
212 if (elf_hwcap2 & HWCAP2_PMULL) {
213 crc32_pmull_algs[0].update = crc32_pmull_update;
214 crc32_pmull_algs[1].update = crc32c_pmull_update;
215
216 if (elf_hwcap2 & HWCAP2_CRC32) {
217 fallback_crc32 = crc32_armv8_le;
218 fallback_crc32c = crc32c_armv8_le;
219 } else {
220 fallback_crc32 = crc32_le;
221 fallback_crc32c = __crc32c_le;
222 }
223 } else if (!(elf_hwcap2 & HWCAP2_CRC32)) {
224 return -ENODEV;
225 }
226
227 return crypto_register_shashes(crc32_pmull_algs,
228 ARRAY_SIZE(crc32_pmull_algs));
229 }
230
231 static void __exit crc32_pmull_mod_exit(void)
232 {
233 crypto_unregister_shashes(crc32_pmull_algs,
234 ARRAY_SIZE(crc32_pmull_algs));
235 }
236
237 static const struct cpu_feature __maybe_unused crc32_cpu_feature[] = {
238 { cpu_feature(CRC32) }, { cpu_feature(PMULL) }, { }
239 };
240 MODULE_DEVICE_TABLE(cpu, crc32_cpu_feature);
241
242 module_init(crc32_pmull_mod_init);
243 module_exit(crc32_pmull_mod_exit);
244
245 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
246 MODULE_LICENSE("GPL v2");
247 MODULE_ALIAS_CRYPTO("crc32");
248 MODULE_ALIAS_CRYPTO("crc32c");