1 | /* |
2 | * Virtio crypto Support |
3 | * |
4 | * Copyright (c) 2016 HUAWEI TECHNOLOGIES CO., LTD. |
5 | * |
6 | * Authors: |
7 | * Gonglei <arei.gonglei@huawei.com> |
8 | * |
9 | * This work is licensed under the terms of the GNU GPL, version 2 or |
10 | * (at your option) any later version. See the COPYING file in the |
11 | * top-level directory. |
12 | */ |
13 | |
14 | #include "qemu/osdep.h" |
15 | #include "qemu/iov.h" |
16 | #include "qemu/main-loop.h" |
17 | #include "qemu/module.h" |
18 | #include "qapi/error.h" |
19 | #include "qemu/error-report.h" |
20 | |
21 | #include "hw/virtio/virtio.h" |
22 | #include "hw/virtio/virtio-crypto.h" |
23 | #include "hw/qdev-properties.h" |
24 | #include "hw/virtio/virtio-access.h" |
25 | #include "standard-headers/linux/virtio_ids.h" |
26 | #include "sysemu/cryptodev-vhost.h" |
27 | |
28 | #define VIRTIO_CRYPTO_VM_VERSION 1 |
29 | |
30 | /* |
31 | * Transfer virtqueue index to crypto queue index. |
32 | * The control virtqueue is after the data virtqueues |
33 | * so the input value doesn't need to be adjusted |
34 | */ |
35 | static inline int virtio_crypto_vq2q(int queue_index) |
36 | { |
37 | return queue_index; |
38 | } |
39 | |
40 | static int |
41 | virtio_crypto_cipher_session_helper(VirtIODevice *vdev, |
42 | CryptoDevBackendSymSessionInfo *info, |
43 | struct virtio_crypto_cipher_session_para *cipher_para, |
44 | struct iovec **iov, unsigned int *out_num) |
45 | { |
46 | VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev); |
47 | unsigned int num = *out_num; |
48 | |
49 | info->cipher_alg = ldl_le_p(&cipher_para->algo); |
50 | info->key_len = ldl_le_p(&cipher_para->keylen); |
51 | info->direction = ldl_le_p(&cipher_para->op); |
52 | DPRINTF("cipher_alg=%" PRIu32 ", info->direction=%" PRIu32 "\n" , |
53 | info->cipher_alg, info->direction); |
54 | |
55 | if (info->key_len > vcrypto->conf.max_cipher_key_len) { |
56 | error_report("virtio-crypto length of cipher key is too big: %u" , |
57 | info->key_len); |
58 | return -VIRTIO_CRYPTO_ERR; |
59 | } |
60 | /* Get cipher key */ |
61 | if (info->key_len > 0) { |
62 | size_t s; |
63 | DPRINTF("keylen=%" PRIu32 "\n" , info->key_len); |
64 | |
65 | info->cipher_key = g_malloc(info->key_len); |
66 | s = iov_to_buf(*iov, num, 0, info->cipher_key, info->key_len); |
67 | if (unlikely(s != info->key_len)) { |
68 | virtio_error(vdev, "virtio-crypto cipher key incorrect" ); |
69 | return -EFAULT; |
70 | } |
71 | iov_discard_front(iov, &num, info->key_len); |
72 | *out_num = num; |
73 | } |
74 | |
75 | return 0; |
76 | } |
77 | |
78 | static int64_t |
79 | virtio_crypto_create_sym_session(VirtIOCrypto *vcrypto, |
80 | struct virtio_crypto_sym_create_session_req *sess_req, |
81 | uint32_t queue_id, |
82 | uint32_t opcode, |
83 | struct iovec *iov, unsigned int out_num) |
84 | { |
85 | VirtIODevice *vdev = VIRTIO_DEVICE(vcrypto); |
86 | CryptoDevBackendSymSessionInfo info; |
87 | int64_t session_id; |
88 | int queue_index; |
89 | uint32_t op_type; |
90 | Error *local_err = NULL; |
91 | int ret; |
92 | |
93 | memset(&info, 0, sizeof(info)); |
94 | op_type = ldl_le_p(&sess_req->op_type); |
95 | info.op_type = op_type; |
96 | info.op_code = opcode; |
97 | |
98 | if (op_type == VIRTIO_CRYPTO_SYM_OP_CIPHER) { |
99 | ret = virtio_crypto_cipher_session_helper(vdev, &info, |
100 | &sess_req->u.cipher.para, |
101 | &iov, &out_num); |
102 | if (ret < 0) { |
103 | goto err; |
104 | } |
105 | } else if (op_type == VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING) { |
106 | size_t s; |
107 | /* cipher part */ |
108 | ret = virtio_crypto_cipher_session_helper(vdev, &info, |
109 | &sess_req->u.chain.para.cipher_param, |
110 | &iov, &out_num); |
111 | if (ret < 0) { |
112 | goto err; |
113 | } |
114 | /* hash part */ |
115 | info.alg_chain_order = ldl_le_p( |
116 | &sess_req->u.chain.para.alg_chain_order); |
117 | info.add_len = ldl_le_p(&sess_req->u.chain.para.aad_len); |
118 | info.hash_mode = ldl_le_p(&sess_req->u.chain.para.hash_mode); |
119 | if (info.hash_mode == VIRTIO_CRYPTO_SYM_HASH_MODE_AUTH) { |
120 | info.hash_alg = ldl_le_p(&sess_req->u.chain.para.u.mac_param.algo); |
121 | info.auth_key_len = ldl_le_p( |
122 | &sess_req->u.chain.para.u.mac_param.auth_key_len); |
123 | info.hash_result_len = ldl_le_p( |
124 | &sess_req->u.chain.para.u.mac_param.hash_result_len); |
125 | if (info.auth_key_len > vcrypto->conf.max_auth_key_len) { |
126 | error_report("virtio-crypto length of auth key is too big: %u" , |
127 | info.auth_key_len); |
128 | ret = -VIRTIO_CRYPTO_ERR; |
129 | goto err; |
130 | } |
131 | /* get auth key */ |
132 | if (info.auth_key_len > 0) { |
133 | DPRINTF("auth_keylen=%" PRIu32 "\n" , info.auth_key_len); |
134 | info.auth_key = g_malloc(info.auth_key_len); |
135 | s = iov_to_buf(iov, out_num, 0, info.auth_key, |
136 | info.auth_key_len); |
137 | if (unlikely(s != info.auth_key_len)) { |
138 | virtio_error(vdev, |
139 | "virtio-crypto authenticated key incorrect" ); |
140 | ret = -EFAULT; |
141 | goto err; |
142 | } |
143 | iov_discard_front(&iov, &out_num, info.auth_key_len); |
144 | } |
145 | } else if (info.hash_mode == VIRTIO_CRYPTO_SYM_HASH_MODE_PLAIN) { |
146 | info.hash_alg = ldl_le_p( |
147 | &sess_req->u.chain.para.u.hash_param.algo); |
148 | info.hash_result_len = ldl_le_p( |
149 | &sess_req->u.chain.para.u.hash_param.hash_result_len); |
150 | } else { |
151 | /* VIRTIO_CRYPTO_SYM_HASH_MODE_NESTED */ |
152 | error_report("unsupported hash mode" ); |
153 | ret = -VIRTIO_CRYPTO_NOTSUPP; |
154 | goto err; |
155 | } |
156 | } else { |
157 | /* VIRTIO_CRYPTO_SYM_OP_NONE */ |
158 | error_report("unsupported cipher op_type: VIRTIO_CRYPTO_SYM_OP_NONE" ); |
159 | ret = -VIRTIO_CRYPTO_NOTSUPP; |
160 | goto err; |
161 | } |
162 | |
163 | queue_index = virtio_crypto_vq2q(queue_id); |
164 | session_id = cryptodev_backend_sym_create_session( |
165 | vcrypto->cryptodev, |
166 | &info, queue_index, &local_err); |
167 | if (session_id >= 0) { |
168 | DPRINTF("create session_id=%" PRIu64 " successfully\n" , |
169 | session_id); |
170 | |
171 | ret = session_id; |
172 | } else { |
173 | if (local_err) { |
174 | error_report_err(local_err); |
175 | } |
176 | ret = -VIRTIO_CRYPTO_ERR; |
177 | } |
178 | |
179 | err: |
180 | g_free(info.cipher_key); |
181 | g_free(info.auth_key); |
182 | return ret; |
183 | } |
184 | |
185 | static uint8_t |
186 | virtio_crypto_handle_close_session(VirtIOCrypto *vcrypto, |
187 | struct virtio_crypto_destroy_session_req *close_sess_req, |
188 | uint32_t queue_id) |
189 | { |
190 | int ret; |
191 | uint64_t session_id; |
192 | uint32_t status; |
193 | Error *local_err = NULL; |
194 | |
195 | session_id = ldq_le_p(&close_sess_req->session_id); |
196 | DPRINTF("close session, id=%" PRIu64 "\n" , session_id); |
197 | |
198 | ret = cryptodev_backend_sym_close_session( |
199 | vcrypto->cryptodev, session_id, queue_id, &local_err); |
200 | if (ret == 0) { |
201 | status = VIRTIO_CRYPTO_OK; |
202 | } else { |
203 | if (local_err) { |
204 | error_report_err(local_err); |
205 | } else { |
206 | error_report("destroy session failed" ); |
207 | } |
208 | status = VIRTIO_CRYPTO_ERR; |
209 | } |
210 | |
211 | return status; |
212 | } |
213 | |
214 | static void virtio_crypto_handle_ctrl(VirtIODevice *vdev, VirtQueue *vq) |
215 | { |
216 | VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev); |
217 | struct virtio_crypto_op_ctrl_req ctrl; |
218 | VirtQueueElement *elem; |
219 | struct iovec *in_iov; |
220 | struct iovec *out_iov; |
221 | unsigned in_num; |
222 | unsigned out_num; |
223 | uint32_t queue_id; |
224 | uint32_t opcode; |
225 | struct virtio_crypto_session_input input; |
226 | int64_t session_id; |
227 | uint8_t status; |
228 | size_t s; |
229 | |
230 | for (;;) { |
231 | elem = virtqueue_pop(vq, sizeof(VirtQueueElement)); |
232 | if (!elem) { |
233 | break; |
234 | } |
235 | if (elem->out_num < 1 || elem->in_num < 1) { |
236 | virtio_error(vdev, "virtio-crypto ctrl missing headers" ); |
237 | virtqueue_detach_element(vq, elem, 0); |
238 | g_free(elem); |
239 | break; |
240 | } |
241 | |
242 | out_num = elem->out_num; |
243 | out_iov = elem->out_sg; |
244 | in_num = elem->in_num; |
245 | in_iov = elem->in_sg; |
246 | if (unlikely(iov_to_buf(out_iov, out_num, 0, &ctrl, sizeof(ctrl)) |
247 | != sizeof(ctrl))) { |
248 | virtio_error(vdev, "virtio-crypto request ctrl_hdr too short" ); |
249 | virtqueue_detach_element(vq, elem, 0); |
250 | g_free(elem); |
251 | break; |
252 | } |
253 | iov_discard_front(&out_iov, &out_num, sizeof(ctrl)); |
254 | |
255 | opcode = ldl_le_p(&ctrl.header.opcode); |
256 | queue_id = ldl_le_p(&ctrl.header.queue_id); |
257 | |
258 | switch (opcode) { |
259 | case VIRTIO_CRYPTO_CIPHER_CREATE_SESSION: |
260 | memset(&input, 0, sizeof(input)); |
261 | session_id = virtio_crypto_create_sym_session(vcrypto, |
262 | &ctrl.u.sym_create_session, |
263 | queue_id, opcode, |
264 | out_iov, out_num); |
265 | /* Serious errors, need to reset virtio crypto device */ |
266 | if (session_id == -EFAULT) { |
267 | virtqueue_detach_element(vq, elem, 0); |
268 | break; |
269 | } else if (session_id == -VIRTIO_CRYPTO_NOTSUPP) { |
270 | stl_le_p(&input.status, VIRTIO_CRYPTO_NOTSUPP); |
271 | } else if (session_id == -VIRTIO_CRYPTO_ERR) { |
272 | stl_le_p(&input.status, VIRTIO_CRYPTO_ERR); |
273 | } else { |
274 | /* Set the session id */ |
275 | stq_le_p(&input.session_id, session_id); |
276 | stl_le_p(&input.status, VIRTIO_CRYPTO_OK); |
277 | } |
278 | |
279 | s = iov_from_buf(in_iov, in_num, 0, &input, sizeof(input)); |
280 | if (unlikely(s != sizeof(input))) { |
281 | virtio_error(vdev, "virtio-crypto input incorrect" ); |
282 | virtqueue_detach_element(vq, elem, 0); |
283 | break; |
284 | } |
285 | virtqueue_push(vq, elem, sizeof(input)); |
286 | virtio_notify(vdev, vq); |
287 | break; |
288 | case VIRTIO_CRYPTO_CIPHER_DESTROY_SESSION: |
289 | case VIRTIO_CRYPTO_HASH_DESTROY_SESSION: |
290 | case VIRTIO_CRYPTO_MAC_DESTROY_SESSION: |
291 | case VIRTIO_CRYPTO_AEAD_DESTROY_SESSION: |
292 | status = virtio_crypto_handle_close_session(vcrypto, |
293 | &ctrl.u.destroy_session, queue_id); |
294 | /* The status only occupy one byte, we can directly use it */ |
295 | s = iov_from_buf(in_iov, in_num, 0, &status, sizeof(status)); |
296 | if (unlikely(s != sizeof(status))) { |
297 | virtio_error(vdev, "virtio-crypto status incorrect" ); |
298 | virtqueue_detach_element(vq, elem, 0); |
299 | break; |
300 | } |
301 | virtqueue_push(vq, elem, sizeof(status)); |
302 | virtio_notify(vdev, vq); |
303 | break; |
304 | case VIRTIO_CRYPTO_HASH_CREATE_SESSION: |
305 | case VIRTIO_CRYPTO_MAC_CREATE_SESSION: |
306 | case VIRTIO_CRYPTO_AEAD_CREATE_SESSION: |
307 | default: |
308 | error_report("virtio-crypto unsupported ctrl opcode: %d" , opcode); |
309 | memset(&input, 0, sizeof(input)); |
310 | stl_le_p(&input.status, VIRTIO_CRYPTO_NOTSUPP); |
311 | s = iov_from_buf(in_iov, in_num, 0, &input, sizeof(input)); |
312 | if (unlikely(s != sizeof(input))) { |
313 | virtio_error(vdev, "virtio-crypto input incorrect" ); |
314 | virtqueue_detach_element(vq, elem, 0); |
315 | break; |
316 | } |
317 | virtqueue_push(vq, elem, sizeof(input)); |
318 | virtio_notify(vdev, vq); |
319 | |
320 | break; |
321 | } /* end switch case */ |
322 | |
323 | g_free(elem); |
324 | } /* end for loop */ |
325 | } |
326 | |
327 | static void virtio_crypto_init_request(VirtIOCrypto *vcrypto, VirtQueue *vq, |
328 | VirtIOCryptoReq *req) |
329 | { |
330 | req->vcrypto = vcrypto; |
331 | req->vq = vq; |
332 | req->in = NULL; |
333 | req->in_iov = NULL; |
334 | req->in_num = 0; |
335 | req->in_len = 0; |
336 | req->flags = CRYPTODEV_BACKEND_ALG__MAX; |
337 | req->u.sym_op_info = NULL; |
338 | } |
339 | |
340 | static void virtio_crypto_free_request(VirtIOCryptoReq *req) |
341 | { |
342 | if (req) { |
343 | if (req->flags == CRYPTODEV_BACKEND_ALG_SYM) { |
344 | size_t max_len; |
345 | CryptoDevBackendSymOpInfo *op_info = req->u.sym_op_info; |
346 | |
347 | max_len = op_info->iv_len + |
348 | op_info->aad_len + |
349 | op_info->src_len + |
350 | op_info->dst_len + |
351 | op_info->digest_result_len; |
352 | |
353 | /* Zeroize and free request data structure */ |
354 | memset(op_info, 0, sizeof(*op_info) + max_len); |
355 | g_free(op_info); |
356 | } |
357 | g_free(req); |
358 | } |
359 | } |
360 | |
361 | static void |
362 | virtio_crypto_sym_input_data_helper(VirtIODevice *vdev, |
363 | VirtIOCryptoReq *req, |
364 | uint32_t status, |
365 | CryptoDevBackendSymOpInfo *sym_op_info) |
366 | { |
367 | size_t s, len; |
368 | |
369 | if (status != VIRTIO_CRYPTO_OK) { |
370 | return; |
371 | } |
372 | |
373 | len = sym_op_info->src_len; |
374 | /* Save the cipher result */ |
375 | s = iov_from_buf(req->in_iov, req->in_num, 0, sym_op_info->dst, len); |
376 | if (s != len) { |
377 | virtio_error(vdev, "virtio-crypto dest data incorrect" ); |
378 | return; |
379 | } |
380 | |
381 | iov_discard_front(&req->in_iov, &req->in_num, len); |
382 | |
383 | if (sym_op_info->op_type == |
384 | VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING) { |
385 | /* Save the digest result */ |
386 | s = iov_from_buf(req->in_iov, req->in_num, 0, |
387 | sym_op_info->digest_result, |
388 | sym_op_info->digest_result_len); |
389 | if (s != sym_op_info->digest_result_len) { |
390 | virtio_error(vdev, "virtio-crypto digest result incorrect" ); |
391 | } |
392 | } |
393 | } |
394 | |
395 | static void virtio_crypto_req_complete(VirtIOCryptoReq *req, uint8_t status) |
396 | { |
397 | VirtIOCrypto *vcrypto = req->vcrypto; |
398 | VirtIODevice *vdev = VIRTIO_DEVICE(vcrypto); |
399 | |
400 | if (req->flags == CRYPTODEV_BACKEND_ALG_SYM) { |
401 | virtio_crypto_sym_input_data_helper(vdev, req, status, |
402 | req->u.sym_op_info); |
403 | } |
404 | stb_p(&req->in->status, status); |
405 | virtqueue_push(req->vq, &req->elem, req->in_len); |
406 | virtio_notify(vdev, req->vq); |
407 | } |
408 | |
409 | static VirtIOCryptoReq * |
410 | virtio_crypto_get_request(VirtIOCrypto *s, VirtQueue *vq) |
411 | { |
412 | VirtIOCryptoReq *req = virtqueue_pop(vq, sizeof(VirtIOCryptoReq)); |
413 | |
414 | if (req) { |
415 | virtio_crypto_init_request(s, vq, req); |
416 | } |
417 | return req; |
418 | } |
419 | |
420 | static CryptoDevBackendSymOpInfo * |
421 | virtio_crypto_sym_op_helper(VirtIODevice *vdev, |
422 | struct virtio_crypto_cipher_para *cipher_para, |
423 | struct virtio_crypto_alg_chain_data_para *alg_chain_para, |
424 | struct iovec *iov, unsigned int out_num) |
425 | { |
426 | VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev); |
427 | CryptoDevBackendSymOpInfo *op_info; |
428 | uint32_t src_len = 0, dst_len = 0; |
429 | uint32_t iv_len = 0; |
430 | uint32_t aad_len = 0, hash_result_len = 0; |
431 | uint32_t hash_start_src_offset = 0, len_to_hash = 0; |
432 | uint32_t cipher_start_src_offset = 0, len_to_cipher = 0; |
433 | |
434 | uint64_t max_len, curr_size = 0; |
435 | size_t s; |
436 | |
437 | /* Plain cipher */ |
438 | if (cipher_para) { |
439 | iv_len = ldl_le_p(&cipher_para->iv_len); |
440 | src_len = ldl_le_p(&cipher_para->src_data_len); |
441 | dst_len = ldl_le_p(&cipher_para->dst_data_len); |
442 | } else if (alg_chain_para) { /* Algorithm chain */ |
443 | iv_len = ldl_le_p(&alg_chain_para->iv_len); |
444 | src_len = ldl_le_p(&alg_chain_para->src_data_len); |
445 | dst_len = ldl_le_p(&alg_chain_para->dst_data_len); |
446 | |
447 | aad_len = ldl_le_p(&alg_chain_para->aad_len); |
448 | hash_result_len = ldl_le_p(&alg_chain_para->hash_result_len); |
449 | hash_start_src_offset = ldl_le_p( |
450 | &alg_chain_para->hash_start_src_offset); |
451 | cipher_start_src_offset = ldl_le_p( |
452 | &alg_chain_para->cipher_start_src_offset); |
453 | len_to_cipher = ldl_le_p(&alg_chain_para->len_to_cipher); |
454 | len_to_hash = ldl_le_p(&alg_chain_para->len_to_hash); |
455 | } else { |
456 | return NULL; |
457 | } |
458 | |
459 | max_len = (uint64_t)iv_len + aad_len + src_len + dst_len + hash_result_len; |
460 | if (unlikely(max_len > vcrypto->conf.max_size)) { |
461 | virtio_error(vdev, "virtio-crypto too big length" ); |
462 | return NULL; |
463 | } |
464 | |
465 | op_info = g_malloc0(sizeof(CryptoDevBackendSymOpInfo) + max_len); |
466 | op_info->iv_len = iv_len; |
467 | op_info->src_len = src_len; |
468 | op_info->dst_len = dst_len; |
469 | op_info->aad_len = aad_len; |
470 | op_info->digest_result_len = hash_result_len; |
471 | op_info->hash_start_src_offset = hash_start_src_offset; |
472 | op_info->len_to_hash = len_to_hash; |
473 | op_info->cipher_start_src_offset = cipher_start_src_offset; |
474 | op_info->len_to_cipher = len_to_cipher; |
475 | /* Handle the initilization vector */ |
476 | if (op_info->iv_len > 0) { |
477 | DPRINTF("iv_len=%" PRIu32 "\n" , op_info->iv_len); |
478 | op_info->iv = op_info->data + curr_size; |
479 | |
480 | s = iov_to_buf(iov, out_num, 0, op_info->iv, op_info->iv_len); |
481 | if (unlikely(s != op_info->iv_len)) { |
482 | virtio_error(vdev, "virtio-crypto iv incorrect" ); |
483 | goto err; |
484 | } |
485 | iov_discard_front(&iov, &out_num, op_info->iv_len); |
486 | curr_size += op_info->iv_len; |
487 | } |
488 | |
489 | /* Handle additional authentication data if exists */ |
490 | if (op_info->aad_len > 0) { |
491 | DPRINTF("aad_len=%" PRIu32 "\n" , op_info->aad_len); |
492 | op_info->aad_data = op_info->data + curr_size; |
493 | |
494 | s = iov_to_buf(iov, out_num, 0, op_info->aad_data, op_info->aad_len); |
495 | if (unlikely(s != op_info->aad_len)) { |
496 | virtio_error(vdev, "virtio-crypto additional auth data incorrect" ); |
497 | goto err; |
498 | } |
499 | iov_discard_front(&iov, &out_num, op_info->aad_len); |
500 | |
501 | curr_size += op_info->aad_len; |
502 | } |
503 | |
504 | /* Handle the source data */ |
505 | if (op_info->src_len > 0) { |
506 | DPRINTF("src_len=%" PRIu32 "\n" , op_info->src_len); |
507 | op_info->src = op_info->data + curr_size; |
508 | |
509 | s = iov_to_buf(iov, out_num, 0, op_info->src, op_info->src_len); |
510 | if (unlikely(s != op_info->src_len)) { |
511 | virtio_error(vdev, "virtio-crypto source data incorrect" ); |
512 | goto err; |
513 | } |
514 | iov_discard_front(&iov, &out_num, op_info->src_len); |
515 | |
516 | curr_size += op_info->src_len; |
517 | } |
518 | |
519 | /* Handle the destination data */ |
520 | op_info->dst = op_info->data + curr_size; |
521 | curr_size += op_info->dst_len; |
522 | |
523 | DPRINTF("dst_len=%" PRIu32 "\n" , op_info->dst_len); |
524 | |
525 | /* Handle the hash digest result */ |
526 | if (hash_result_len > 0) { |
527 | DPRINTF("hash_result_len=%" PRIu32 "\n" , hash_result_len); |
528 | op_info->digest_result = op_info->data + curr_size; |
529 | } |
530 | |
531 | return op_info; |
532 | |
533 | err: |
534 | g_free(op_info); |
535 | return NULL; |
536 | } |
537 | |
538 | static int |
539 | virtio_crypto_handle_sym_req(VirtIOCrypto *vcrypto, |
540 | struct virtio_crypto_sym_data_req *req, |
541 | CryptoDevBackendSymOpInfo **sym_op_info, |
542 | struct iovec *iov, unsigned int out_num) |
543 | { |
544 | VirtIODevice *vdev = VIRTIO_DEVICE(vcrypto); |
545 | uint32_t op_type; |
546 | CryptoDevBackendSymOpInfo *op_info; |
547 | |
548 | op_type = ldl_le_p(&req->op_type); |
549 | |
550 | if (op_type == VIRTIO_CRYPTO_SYM_OP_CIPHER) { |
551 | op_info = virtio_crypto_sym_op_helper(vdev, &req->u.cipher.para, |
552 | NULL, iov, out_num); |
553 | if (!op_info) { |
554 | return -EFAULT; |
555 | } |
556 | op_info->op_type = op_type; |
557 | } else if (op_type == VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING) { |
558 | op_info = virtio_crypto_sym_op_helper(vdev, NULL, |
559 | &req->u.chain.para, |
560 | iov, out_num); |
561 | if (!op_info) { |
562 | return -EFAULT; |
563 | } |
564 | op_info->op_type = op_type; |
565 | } else { |
566 | /* VIRTIO_CRYPTO_SYM_OP_NONE */ |
567 | error_report("virtio-crypto unsupported cipher type" ); |
568 | return -VIRTIO_CRYPTO_NOTSUPP; |
569 | } |
570 | |
571 | *sym_op_info = op_info; |
572 | |
573 | return 0; |
574 | } |
575 | |
576 | static int |
577 | virtio_crypto_handle_request(VirtIOCryptoReq *request) |
578 | { |
579 | VirtIOCrypto *vcrypto = request->vcrypto; |
580 | VirtIODevice *vdev = VIRTIO_DEVICE(vcrypto); |
581 | VirtQueueElement *elem = &request->elem; |
582 | int queue_index = virtio_crypto_vq2q(virtio_get_queue_index(request->vq)); |
583 | struct virtio_crypto_op_data_req req; |
584 | int ret; |
585 | struct iovec *in_iov; |
586 | struct iovec *out_iov; |
587 | unsigned in_num; |
588 | unsigned out_num; |
589 | uint32_t opcode; |
590 | uint8_t status = VIRTIO_CRYPTO_ERR; |
591 | uint64_t session_id; |
592 | CryptoDevBackendSymOpInfo *sym_op_info = NULL; |
593 | Error *local_err = NULL; |
594 | |
595 | if (elem->out_num < 1 || elem->in_num < 1) { |
596 | virtio_error(vdev, "virtio-crypto dataq missing headers" ); |
597 | return -1; |
598 | } |
599 | |
600 | out_num = elem->out_num; |
601 | out_iov = elem->out_sg; |
602 | in_num = elem->in_num; |
603 | in_iov = elem->in_sg; |
604 | if (unlikely(iov_to_buf(out_iov, out_num, 0, &req, sizeof(req)) |
605 | != sizeof(req))) { |
606 | virtio_error(vdev, "virtio-crypto request outhdr too short" ); |
607 | return -1; |
608 | } |
609 | iov_discard_front(&out_iov, &out_num, sizeof(req)); |
610 | |
611 | if (in_iov[in_num - 1].iov_len < |
612 | sizeof(struct virtio_crypto_inhdr)) { |
613 | virtio_error(vdev, "virtio-crypto request inhdr too short" ); |
614 | return -1; |
615 | } |
616 | /* We always touch the last byte, so just see how big in_iov is. */ |
617 | request->in_len = iov_size(in_iov, in_num); |
618 | request->in = (void *)in_iov[in_num - 1].iov_base |
619 | + in_iov[in_num - 1].iov_len |
620 | - sizeof(struct virtio_crypto_inhdr); |
621 | iov_discard_back(in_iov, &in_num, sizeof(struct virtio_crypto_inhdr)); |
622 | |
623 | /* |
624 | * The length of operation result, including dest_data |
625 | * and digest_result if exists. |
626 | */ |
627 | request->in_num = in_num; |
628 | request->in_iov = in_iov; |
629 | |
630 | opcode = ldl_le_p(&req.header.opcode); |
631 | session_id = ldq_le_p(&req.header.session_id); |
632 | |
633 | switch (opcode) { |
634 | case VIRTIO_CRYPTO_CIPHER_ENCRYPT: |
635 | case VIRTIO_CRYPTO_CIPHER_DECRYPT: |
636 | ret = virtio_crypto_handle_sym_req(vcrypto, |
637 | &req.u.sym_req, |
638 | &sym_op_info, |
639 | out_iov, out_num); |
640 | /* Serious errors, need to reset virtio crypto device */ |
641 | if (ret == -EFAULT) { |
642 | return -1; |
643 | } else if (ret == -VIRTIO_CRYPTO_NOTSUPP) { |
644 | virtio_crypto_req_complete(request, VIRTIO_CRYPTO_NOTSUPP); |
645 | virtio_crypto_free_request(request); |
646 | } else { |
647 | sym_op_info->session_id = session_id; |
648 | |
649 | /* Set request's parameter */ |
650 | request->flags = CRYPTODEV_BACKEND_ALG_SYM; |
651 | request->u.sym_op_info = sym_op_info; |
652 | ret = cryptodev_backend_crypto_operation(vcrypto->cryptodev, |
653 | request, queue_index, &local_err); |
654 | if (ret < 0) { |
655 | status = -ret; |
656 | if (local_err) { |
657 | error_report_err(local_err); |
658 | } |
659 | } else { /* ret == VIRTIO_CRYPTO_OK */ |
660 | status = ret; |
661 | } |
662 | virtio_crypto_req_complete(request, status); |
663 | virtio_crypto_free_request(request); |
664 | } |
665 | break; |
666 | case VIRTIO_CRYPTO_HASH: |
667 | case VIRTIO_CRYPTO_MAC: |
668 | case VIRTIO_CRYPTO_AEAD_ENCRYPT: |
669 | case VIRTIO_CRYPTO_AEAD_DECRYPT: |
670 | default: |
671 | error_report("virtio-crypto unsupported dataq opcode: %u" , |
672 | opcode); |
673 | virtio_crypto_req_complete(request, VIRTIO_CRYPTO_NOTSUPP); |
674 | virtio_crypto_free_request(request); |
675 | } |
676 | |
677 | return 0; |
678 | } |
679 | |
680 | static void virtio_crypto_handle_dataq(VirtIODevice *vdev, VirtQueue *vq) |
681 | { |
682 | VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev); |
683 | VirtIOCryptoReq *req; |
684 | |
685 | while ((req = virtio_crypto_get_request(vcrypto, vq))) { |
686 | if (virtio_crypto_handle_request(req) < 0) { |
687 | virtqueue_detach_element(req->vq, &req->elem, 0); |
688 | virtio_crypto_free_request(req); |
689 | break; |
690 | } |
691 | } |
692 | } |
693 | |
694 | static void virtio_crypto_dataq_bh(void *opaque) |
695 | { |
696 | VirtIOCryptoQueue *q = opaque; |
697 | VirtIOCrypto *vcrypto = q->vcrypto; |
698 | VirtIODevice *vdev = VIRTIO_DEVICE(vcrypto); |
699 | |
700 | /* This happens when device was stopped but BH wasn't. */ |
701 | if (!vdev->vm_running) { |
702 | return; |
703 | } |
704 | |
705 | /* Just in case the driver is not ready on more */ |
706 | if (unlikely(!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK))) { |
707 | return; |
708 | } |
709 | |
710 | for (;;) { |
711 | virtio_crypto_handle_dataq(vdev, q->dataq); |
712 | virtio_queue_set_notification(q->dataq, 1); |
713 | |
714 | /* Are we done or did the guest add more buffers? */ |
715 | if (virtio_queue_empty(q->dataq)) { |
716 | break; |
717 | } |
718 | |
719 | virtio_queue_set_notification(q->dataq, 0); |
720 | } |
721 | } |
722 | |
723 | static void |
724 | virtio_crypto_handle_dataq_bh(VirtIODevice *vdev, VirtQueue *vq) |
725 | { |
726 | VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev); |
727 | VirtIOCryptoQueue *q = |
728 | &vcrypto->vqs[virtio_crypto_vq2q(virtio_get_queue_index(vq))]; |
729 | |
730 | /* This happens when device was stopped but VCPU wasn't. */ |
731 | if (!vdev->vm_running) { |
732 | return; |
733 | } |
734 | virtio_queue_set_notification(vq, 0); |
735 | qemu_bh_schedule(q->dataq_bh); |
736 | } |
737 | |
738 | static uint64_t virtio_crypto_get_features(VirtIODevice *vdev, |
739 | uint64_t features, |
740 | Error **errp) |
741 | { |
742 | return features; |
743 | } |
744 | |
745 | static void virtio_crypto_reset(VirtIODevice *vdev) |
746 | { |
747 | VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev); |
748 | /* multiqueue is disabled by default */ |
749 | vcrypto->curr_queues = 1; |
750 | if (!cryptodev_backend_is_ready(vcrypto->cryptodev)) { |
751 | vcrypto->status &= ~VIRTIO_CRYPTO_S_HW_READY; |
752 | } else { |
753 | vcrypto->status |= VIRTIO_CRYPTO_S_HW_READY; |
754 | } |
755 | } |
756 | |
757 | static void virtio_crypto_init_config(VirtIODevice *vdev) |
758 | { |
759 | VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev); |
760 | |
761 | vcrypto->conf.crypto_services = |
762 | vcrypto->conf.cryptodev->conf.crypto_services; |
763 | vcrypto->conf.cipher_algo_l = |
764 | vcrypto->conf.cryptodev->conf.cipher_algo_l; |
765 | vcrypto->conf.cipher_algo_h = |
766 | vcrypto->conf.cryptodev->conf.cipher_algo_h; |
767 | vcrypto->conf.hash_algo = vcrypto->conf.cryptodev->conf.hash_algo; |
768 | vcrypto->conf.mac_algo_l = vcrypto->conf.cryptodev->conf.mac_algo_l; |
769 | vcrypto->conf.mac_algo_h = vcrypto->conf.cryptodev->conf.mac_algo_h; |
770 | vcrypto->conf.aead_algo = vcrypto->conf.cryptodev->conf.aead_algo; |
771 | vcrypto->conf.max_cipher_key_len = |
772 | vcrypto->conf.cryptodev->conf.max_cipher_key_len; |
773 | vcrypto->conf.max_auth_key_len = |
774 | vcrypto->conf.cryptodev->conf.max_auth_key_len; |
775 | vcrypto->conf.max_size = vcrypto->conf.cryptodev->conf.max_size; |
776 | } |
777 | |
778 | static void virtio_crypto_device_realize(DeviceState *dev, Error **errp) |
779 | { |
780 | VirtIODevice *vdev = VIRTIO_DEVICE(dev); |
781 | VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(dev); |
782 | int i; |
783 | |
784 | vcrypto->cryptodev = vcrypto->conf.cryptodev; |
785 | if (vcrypto->cryptodev == NULL) { |
786 | error_setg(errp, "'cryptodev' parameter expects a valid object" ); |
787 | return; |
788 | } else if (cryptodev_backend_is_used(vcrypto->cryptodev)) { |
789 | char *path = object_get_canonical_path_component(OBJECT(vcrypto->conf.cryptodev)); |
790 | error_setg(errp, "can't use already used cryptodev backend: %s" , path); |
791 | g_free(path); |
792 | return; |
793 | } |
794 | |
795 | vcrypto->max_queues = MAX(vcrypto->cryptodev->conf.peers.queues, 1); |
796 | if (vcrypto->max_queues + 1 > VIRTIO_QUEUE_MAX) { |
797 | error_setg(errp, "Invalid number of queues (= %" PRIu32 "), " |
798 | "must be a positive integer less than %d." , |
799 | vcrypto->max_queues, VIRTIO_QUEUE_MAX); |
800 | return; |
801 | } |
802 | |
803 | virtio_init(vdev, "virtio-crypto" , VIRTIO_ID_CRYPTO, vcrypto->config_size); |
804 | vcrypto->curr_queues = 1; |
805 | vcrypto->vqs = g_malloc0(sizeof(VirtIOCryptoQueue) * vcrypto->max_queues); |
806 | for (i = 0; i < vcrypto->max_queues; i++) { |
807 | vcrypto->vqs[i].dataq = |
808 | virtio_add_queue(vdev, 1024, virtio_crypto_handle_dataq_bh); |
809 | vcrypto->vqs[i].dataq_bh = |
810 | qemu_bh_new(virtio_crypto_dataq_bh, &vcrypto->vqs[i]); |
811 | vcrypto->vqs[i].vcrypto = vcrypto; |
812 | } |
813 | |
814 | vcrypto->ctrl_vq = virtio_add_queue(vdev, 64, virtio_crypto_handle_ctrl); |
815 | if (!cryptodev_backend_is_ready(vcrypto->cryptodev)) { |
816 | vcrypto->status &= ~VIRTIO_CRYPTO_S_HW_READY; |
817 | } else { |
818 | vcrypto->status |= VIRTIO_CRYPTO_S_HW_READY; |
819 | } |
820 | |
821 | virtio_crypto_init_config(vdev); |
822 | cryptodev_backend_set_used(vcrypto->cryptodev, true); |
823 | } |
824 | |
825 | static void virtio_crypto_device_unrealize(DeviceState *dev, Error **errp) |
826 | { |
827 | VirtIODevice *vdev = VIRTIO_DEVICE(dev); |
828 | VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(dev); |
829 | VirtIOCryptoQueue *q; |
830 | int i, max_queues; |
831 | |
832 | max_queues = vcrypto->multiqueue ? vcrypto->max_queues : 1; |
833 | for (i = 0; i < max_queues; i++) { |
834 | virtio_del_queue(vdev, i); |
835 | q = &vcrypto->vqs[i]; |
836 | qemu_bh_delete(q->dataq_bh); |
837 | } |
838 | |
839 | g_free(vcrypto->vqs); |
840 | |
841 | virtio_cleanup(vdev); |
842 | cryptodev_backend_set_used(vcrypto->cryptodev, false); |
843 | } |
844 | |
845 | static const VMStateDescription vmstate_virtio_crypto = { |
846 | .name = "virtio-crypto" , |
847 | .unmigratable = 1, |
848 | .minimum_version_id = VIRTIO_CRYPTO_VM_VERSION, |
849 | .version_id = VIRTIO_CRYPTO_VM_VERSION, |
850 | .fields = (VMStateField[]) { |
851 | VMSTATE_VIRTIO_DEVICE, |
852 | VMSTATE_END_OF_LIST() |
853 | }, |
854 | }; |
855 | |
856 | static Property virtio_crypto_properties[] = { |
857 | DEFINE_PROP_LINK("cryptodev" , VirtIOCrypto, conf.cryptodev, |
858 | TYPE_CRYPTODEV_BACKEND, CryptoDevBackend *), |
859 | DEFINE_PROP_END_OF_LIST(), |
860 | }; |
861 | |
862 | static void virtio_crypto_get_config(VirtIODevice *vdev, uint8_t *config) |
863 | { |
864 | VirtIOCrypto *c = VIRTIO_CRYPTO(vdev); |
865 | struct virtio_crypto_config crypto_cfg = {}; |
866 | |
867 | /* |
868 | * Virtio-crypto device conforms to VIRTIO 1.0 which is always LE, |
869 | * so we can use LE accessors directly. |
870 | */ |
871 | stl_le_p(&crypto_cfg.status, c->status); |
872 | stl_le_p(&crypto_cfg.max_dataqueues, c->max_queues); |
873 | stl_le_p(&crypto_cfg.crypto_services, c->conf.crypto_services); |
874 | stl_le_p(&crypto_cfg.cipher_algo_l, c->conf.cipher_algo_l); |
875 | stl_le_p(&crypto_cfg.cipher_algo_h, c->conf.cipher_algo_h); |
876 | stl_le_p(&crypto_cfg.hash_algo, c->conf.hash_algo); |
877 | stl_le_p(&crypto_cfg.mac_algo_l, c->conf.mac_algo_l); |
878 | stl_le_p(&crypto_cfg.mac_algo_h, c->conf.mac_algo_h); |
879 | stl_le_p(&crypto_cfg.aead_algo, c->conf.aead_algo); |
880 | stl_le_p(&crypto_cfg.max_cipher_key_len, c->conf.max_cipher_key_len); |
881 | stl_le_p(&crypto_cfg.max_auth_key_len, c->conf.max_auth_key_len); |
882 | stq_le_p(&crypto_cfg.max_size, c->conf.max_size); |
883 | |
884 | memcpy(config, &crypto_cfg, c->config_size); |
885 | } |
886 | |
887 | static bool virtio_crypto_started(VirtIOCrypto *c, uint8_t status) |
888 | { |
889 | VirtIODevice *vdev = VIRTIO_DEVICE(c); |
890 | return (status & VIRTIO_CONFIG_S_DRIVER_OK) && |
891 | (c->status & VIRTIO_CRYPTO_S_HW_READY) && vdev->vm_running; |
892 | } |
893 | |
894 | static void virtio_crypto_vhost_status(VirtIOCrypto *c, uint8_t status) |
895 | { |
896 | VirtIODevice *vdev = VIRTIO_DEVICE(c); |
897 | int queues = c->multiqueue ? c->max_queues : 1; |
898 | CryptoDevBackend *b = c->cryptodev; |
899 | CryptoDevBackendClient *cc = b->conf.peers.ccs[0]; |
900 | |
901 | if (!cryptodev_get_vhost(cc, b, 0)) { |
902 | return; |
903 | } |
904 | |
905 | if ((virtio_crypto_started(c, status)) == !!c->vhost_started) { |
906 | return; |
907 | } |
908 | |
909 | if (!c->vhost_started) { |
910 | int r; |
911 | |
912 | c->vhost_started = 1; |
913 | r = cryptodev_vhost_start(vdev, queues); |
914 | if (r < 0) { |
915 | error_report("unable to start vhost crypto: %d: " |
916 | "falling back on userspace virtio" , -r); |
917 | c->vhost_started = 0; |
918 | } |
919 | } else { |
920 | cryptodev_vhost_stop(vdev, queues); |
921 | c->vhost_started = 0; |
922 | } |
923 | } |
924 | |
925 | static void virtio_crypto_set_status(VirtIODevice *vdev, uint8_t status) |
926 | { |
927 | VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev); |
928 | |
929 | virtio_crypto_vhost_status(vcrypto, status); |
930 | } |
931 | |
932 | static void virtio_crypto_guest_notifier_mask(VirtIODevice *vdev, int idx, |
933 | bool mask) |
934 | { |
935 | VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev); |
936 | int queue = virtio_crypto_vq2q(idx); |
937 | |
938 | assert(vcrypto->vhost_started); |
939 | |
940 | cryptodev_vhost_virtqueue_mask(vdev, queue, idx, mask); |
941 | } |
942 | |
943 | static bool virtio_crypto_guest_notifier_pending(VirtIODevice *vdev, int idx) |
944 | { |
945 | VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev); |
946 | int queue = virtio_crypto_vq2q(idx); |
947 | |
948 | assert(vcrypto->vhost_started); |
949 | |
950 | return cryptodev_vhost_virtqueue_pending(vdev, queue, idx); |
951 | } |
952 | |
953 | static void virtio_crypto_class_init(ObjectClass *klass, void *data) |
954 | { |
955 | DeviceClass *dc = DEVICE_CLASS(klass); |
956 | VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass); |
957 | |
958 | dc->props = virtio_crypto_properties; |
959 | dc->vmsd = &vmstate_virtio_crypto; |
960 | set_bit(DEVICE_CATEGORY_MISC, dc->categories); |
961 | vdc->realize = virtio_crypto_device_realize; |
962 | vdc->unrealize = virtio_crypto_device_unrealize; |
963 | vdc->get_config = virtio_crypto_get_config; |
964 | vdc->get_features = virtio_crypto_get_features; |
965 | vdc->reset = virtio_crypto_reset; |
966 | vdc->set_status = virtio_crypto_set_status; |
967 | vdc->guest_notifier_mask = virtio_crypto_guest_notifier_mask; |
968 | vdc->guest_notifier_pending = virtio_crypto_guest_notifier_pending; |
969 | } |
970 | |
971 | static void virtio_crypto_instance_init(Object *obj) |
972 | { |
973 | VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(obj); |
974 | |
975 | /* |
976 | * The default config_size is sizeof(struct virtio_crypto_config). |
977 | * Can be overriden with virtio_crypto_set_config_size. |
978 | */ |
979 | vcrypto->config_size = sizeof(struct virtio_crypto_config); |
980 | } |
981 | |
982 | static const TypeInfo virtio_crypto_info = { |
983 | .name = TYPE_VIRTIO_CRYPTO, |
984 | .parent = TYPE_VIRTIO_DEVICE, |
985 | .instance_size = sizeof(VirtIOCrypto), |
986 | .instance_init = virtio_crypto_instance_init, |
987 | .class_init = virtio_crypto_class_init, |
988 | }; |
989 | |
990 | static void virtio_register_types(void) |
991 | { |
992 | type_register_static(&virtio_crypto_info); |
993 | } |
994 | |
995 | type_init(virtio_register_types) |
996 | |