crypto/qat: add device files
[dpdk.git] / drivers / crypto / virtio / virtio_cryptodev.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2018 HUAWEI TECHNOLOGIES CO., LTD.
3  */
4 #include <stdbool.h>
5 #include <unistd.h>
6
7 #include <rte_common.h>
8 #include <rte_errno.h>
9 #include <rte_pci.h>
10 #include <rte_bus_pci.h>
11 #include <rte_cryptodev.h>
12 #include <rte_cryptodev_pmd.h>
13 #include <rte_eal.h>
14
15 #include "virtio_cryptodev.h"
16 #include "virtqueue.h"
17 #include "virtio_crypto_algs.h"
18 #include "virtio_crypto_capabilities.h"
19
20 int virtio_crypto_logtype_init;
21 int virtio_crypto_logtype_session;
22 int virtio_crypto_logtype_rx;
23 int virtio_crypto_logtype_tx;
24 int virtio_crypto_logtype_driver;
25
26 static int virtio_crypto_dev_configure(struct rte_cryptodev *dev,
27                 struct rte_cryptodev_config *config);
28 static int virtio_crypto_dev_start(struct rte_cryptodev *dev);
29 static void virtio_crypto_dev_stop(struct rte_cryptodev *dev);
30 static int virtio_crypto_dev_close(struct rte_cryptodev *dev);
31 static void virtio_crypto_dev_info_get(struct rte_cryptodev *dev,
32                 struct rte_cryptodev_info *dev_info);
33 static void virtio_crypto_dev_stats_get(struct rte_cryptodev *dev,
34                 struct rte_cryptodev_stats *stats);
35 static void virtio_crypto_dev_stats_reset(struct rte_cryptodev *dev);
36 static int virtio_crypto_qp_setup(struct rte_cryptodev *dev,
37                 uint16_t queue_pair_id,
38                 const struct rte_cryptodev_qp_conf *qp_conf,
39                 int socket_id,
40                 struct rte_mempool *session_pool);
41 static int virtio_crypto_qp_release(struct rte_cryptodev *dev,
42                 uint16_t queue_pair_id);
43 static void virtio_crypto_dev_free_mbufs(struct rte_cryptodev *dev);
44 static unsigned int virtio_crypto_sym_get_session_private_size(
45                 struct rte_cryptodev *dev);
46 static void virtio_crypto_sym_clear_session(struct rte_cryptodev *dev,
47                 struct rte_cryptodev_sym_session *sess);
48 static int virtio_crypto_sym_configure_session(struct rte_cryptodev *dev,
49                 struct rte_crypto_sym_xform *xform,
50                 struct rte_cryptodev_sym_session *session,
51                 struct rte_mempool *mp);
52
53 /*
54  * The set of PCI devices this driver supports
55  */
56 static const struct rte_pci_id pci_id_virtio_crypto_map[] = {
57         { RTE_PCI_DEVICE(VIRTIO_CRYPTO_PCI_VENDORID,
58                                 VIRTIO_CRYPTO_PCI_DEVICEID) },
59         { .vendor_id = 0, /* sentinel */ },
60 };
61
62 static const struct rte_cryptodev_capabilities virtio_capabilities[] = {
63         VIRTIO_SYM_CAPABILITIES,
64         RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
65 };
66
67 uint8_t cryptodev_virtio_driver_id;
68
69 #define NUM_ENTRY_SYM_CREATE_SESSION 4
70
71 static int
72 virtio_crypto_send_command(struct virtqueue *vq,
73                 struct virtio_crypto_op_ctrl_req *ctrl, uint8_t *cipher_key,
74                 uint8_t *auth_key, struct virtio_crypto_session *session)
75 {
76         uint8_t idx = 0;
77         uint8_t needed = 1;
78         uint32_t head = 0;
79         uint32_t len_cipher_key = 0;
80         uint32_t len_auth_key = 0;
81         uint32_t len_ctrl_req = sizeof(struct virtio_crypto_op_ctrl_req);
82         uint32_t len_session_input = sizeof(struct virtio_crypto_session_input);
83         uint32_t len_total = 0;
84         uint32_t input_offset = 0;
85         void *virt_addr_started = NULL;
86         phys_addr_t phys_addr_started;
87         struct vring_desc *desc;
88         uint32_t desc_offset;
89         struct virtio_crypto_session_input *input;
90         int ret;
91
92         PMD_INIT_FUNC_TRACE();
93
94         if (session == NULL) {
95                 VIRTIO_CRYPTO_SESSION_LOG_ERR("session is NULL.");
96                 return -EINVAL;
97         }
98         /* cipher only is supported, it is available if auth_key is NULL */
99         if (!cipher_key) {
100                 VIRTIO_CRYPTO_SESSION_LOG_ERR("cipher key is NULL.");
101                 return -EINVAL;
102         }
103
104         head = vq->vq_desc_head_idx;
105         VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_desc_head_idx = %d, vq = %p",
106                                         head, vq);
107
108         if (vq->vq_free_cnt < needed) {
109                 VIRTIO_CRYPTO_SESSION_LOG_ERR("Not enough entry");
110                 return -ENOSPC;
111         }
112
113         /* calculate the length of cipher key */
114         if (cipher_key) {
115                 switch (ctrl->u.sym_create_session.op_type) {
116                 case VIRTIO_CRYPTO_SYM_OP_CIPHER:
117                         len_cipher_key
118                                 = ctrl->u.sym_create_session.u.cipher
119                                                         .para.keylen;
120                         break;
121                 case VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING:
122                         len_cipher_key
123                                 = ctrl->u.sym_create_session.u.chain
124                                         .para.cipher_param.keylen;
125                         break;
126                 default:
127                         VIRTIO_CRYPTO_SESSION_LOG_ERR("invalid op type");
128                         return -EINVAL;
129                 }
130         }
131
132         /* calculate the length of auth key */
133         if (auth_key) {
134                 len_auth_key =
135                         ctrl->u.sym_create_session.u.chain.para.u.mac_param
136                                 .auth_key_len;
137         }
138
139         /*
140          * malloc memory to store indirect vring_desc entries, including
141          * ctrl request, cipher key, auth key, session input and desc vring
142          */
143         desc_offset = len_ctrl_req + len_cipher_key + len_auth_key
144                 + len_session_input;
145         virt_addr_started = rte_malloc(NULL,
146                 desc_offset + NUM_ENTRY_SYM_CREATE_SESSION
147                         * sizeof(struct vring_desc), RTE_CACHE_LINE_SIZE);
148         if (virt_addr_started == NULL) {
149                 VIRTIO_CRYPTO_SESSION_LOG_ERR("not enough heap memory");
150                 return -ENOSPC;
151         }
152         phys_addr_started = rte_malloc_virt2iova(virt_addr_started);
153
154         /* address to store indirect vring desc entries */
155         desc = (struct vring_desc *)
156                 ((uint8_t *)virt_addr_started + desc_offset);
157
158         /*  ctrl req part */
159         memcpy(virt_addr_started, ctrl, len_ctrl_req);
160         desc[idx].addr = phys_addr_started;
161         desc[idx].len = len_ctrl_req;
162         desc[idx].flags = VRING_DESC_F_NEXT;
163         desc[idx].next = idx + 1;
164         idx++;
165         len_total += len_ctrl_req;
166         input_offset += len_ctrl_req;
167
168         /* cipher key part */
169         if (len_cipher_key > 0) {
170                 memcpy((uint8_t *)virt_addr_started + len_total,
171                         cipher_key, len_cipher_key);
172
173                 desc[idx].addr = phys_addr_started + len_total;
174                 desc[idx].len = len_cipher_key;
175                 desc[idx].flags = VRING_DESC_F_NEXT;
176                 desc[idx].next = idx + 1;
177                 idx++;
178                 len_total += len_cipher_key;
179                 input_offset += len_cipher_key;
180         }
181
182         /* auth key part */
183         if (len_auth_key > 0) {
184                 memcpy((uint8_t *)virt_addr_started + len_total,
185                         auth_key, len_auth_key);
186
187                 desc[idx].addr = phys_addr_started + len_total;
188                 desc[idx].len = len_auth_key;
189                 desc[idx].flags = VRING_DESC_F_NEXT;
190                 desc[idx].next = idx + 1;
191                 idx++;
192                 len_total += len_auth_key;
193                 input_offset += len_auth_key;
194         }
195
196         /* input part */
197         input = (struct virtio_crypto_session_input *)
198                 ((uint8_t *)virt_addr_started + input_offset);
199         input->status = VIRTIO_CRYPTO_ERR;
200         input->session_id = ~0ULL;
201         desc[idx].addr = phys_addr_started + len_total;
202         desc[idx].len = len_session_input;
203         desc[idx].flags = VRING_DESC_F_WRITE;
204         idx++;
205
206         /* use a single desc entry */
207         vq->vq_ring.desc[head].addr = phys_addr_started + desc_offset;
208         vq->vq_ring.desc[head].len = idx * sizeof(struct vring_desc);
209         vq->vq_ring.desc[head].flags = VRING_DESC_F_INDIRECT;
210         vq->vq_free_cnt--;
211
212         vq->vq_desc_head_idx = vq->vq_ring.desc[head].next;
213
214         vq_update_avail_ring(vq, head);
215         vq_update_avail_idx(vq);
216
217         VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_queue_index = %d",
218                                         vq->vq_queue_index);
219
220         virtqueue_notify(vq);
221
222         rte_rmb();
223         while (vq->vq_used_cons_idx == vq->vq_ring.used->idx) {
224                 rte_rmb();
225                 usleep(100);
226         }
227
228         while (vq->vq_used_cons_idx != vq->vq_ring.used->idx) {
229                 uint32_t idx, desc_idx, used_idx;
230                 struct vring_used_elem *uep;
231
232                 used_idx = (uint32_t)(vq->vq_used_cons_idx
233                                 & (vq->vq_nentries - 1));
234                 uep = &vq->vq_ring.used->ring[used_idx];
235                 idx = (uint32_t) uep->id;
236                 desc_idx = idx;
237
238                 while (vq->vq_ring.desc[desc_idx].flags & VRING_DESC_F_NEXT) {
239                         desc_idx = vq->vq_ring.desc[desc_idx].next;
240                         vq->vq_free_cnt++;
241                 }
242
243                 vq->vq_ring.desc[desc_idx].next = vq->vq_desc_head_idx;
244                 vq->vq_desc_head_idx = idx;
245
246                 vq->vq_used_cons_idx++;
247                 vq->vq_free_cnt++;
248         }
249
250         VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_free_cnt=%d\n"
251                         "vq->vq_desc_head_idx=%d",
252                         vq->vq_free_cnt, vq->vq_desc_head_idx);
253
254         /* get the result */
255         if (input->status != VIRTIO_CRYPTO_OK) {
256                 VIRTIO_CRYPTO_SESSION_LOG_ERR("Something wrong on backend! "
257                                 "status=%u, session_id=%" PRIu64 "",
258                                 input->status, input->session_id);
259                 rte_free(virt_addr_started);
260                 ret = -1;
261         } else {
262                 session->session_id = input->session_id;
263
264                 VIRTIO_CRYPTO_SESSION_LOG_INFO("Create session successfully, "
265                                 "session_id=%" PRIu64 "", input->session_id);
266                 rte_free(virt_addr_started);
267                 ret = 0;
268         }
269
270         return ret;
271 }
272
273 void
274 virtio_crypto_queue_release(struct virtqueue *vq)
275 {
276         struct virtio_crypto_hw *hw;
277
278         PMD_INIT_FUNC_TRACE();
279
280         if (vq) {
281                 hw = vq->hw;
282                 /* Select and deactivate the queue */
283                 VTPCI_OPS(hw)->del_queue(hw, vq);
284
285                 rte_memzone_free(vq->mz);
286                 rte_mempool_free(vq->mpool);
287                 rte_free(vq);
288         }
289 }
290
291 #define MPOOL_MAX_NAME_SZ 32
292
293 int
294 virtio_crypto_queue_setup(struct rte_cryptodev *dev,
295                 int queue_type,
296                 uint16_t vtpci_queue_idx,
297                 uint16_t nb_desc,
298                 int socket_id,
299                 struct virtqueue **pvq)
300 {
301         char vq_name[VIRTQUEUE_MAX_NAME_SZ];
302         char mpool_name[MPOOL_MAX_NAME_SZ];
303         const struct rte_memzone *mz;
304         unsigned int vq_size, size;
305         struct virtio_crypto_hw *hw = dev->data->dev_private;
306         struct virtqueue *vq = NULL;
307         uint32_t i = 0;
308         uint32_t j;
309
310         PMD_INIT_FUNC_TRACE();
311
312         VIRTIO_CRYPTO_INIT_LOG_DBG("setting up queue: %u", vtpci_queue_idx);
313
314         /*
315          * Read the virtqueue size from the Queue Size field
316          * Always power of 2 and if 0 virtqueue does not exist
317          */
318         vq_size = VTPCI_OPS(hw)->get_queue_num(hw, vtpci_queue_idx);
319         if (vq_size == 0) {
320                 VIRTIO_CRYPTO_INIT_LOG_ERR("virtqueue does not exist");
321                 return -EINVAL;
322         }
323         VIRTIO_CRYPTO_INIT_LOG_DBG("vq_size: %u", vq_size);
324
325         if (!rte_is_power_of_2(vq_size)) {
326                 VIRTIO_CRYPTO_INIT_LOG_ERR("virtqueue size is not powerof 2");
327                 return -EINVAL;
328         }
329
330         if (queue_type == VTCRYPTO_DATAQ) {
331                 snprintf(vq_name, sizeof(vq_name), "dev%d_dataqueue%d",
332                                 dev->data->dev_id, vtpci_queue_idx);
333                 snprintf(mpool_name, sizeof(mpool_name),
334                                 "dev%d_dataqueue%d_mpool",
335                                 dev->data->dev_id, vtpci_queue_idx);
336         } else if (queue_type == VTCRYPTO_CTRLQ) {
337                 snprintf(vq_name, sizeof(vq_name), "dev%d_controlqueue",
338                                 dev->data->dev_id);
339                 snprintf(mpool_name, sizeof(mpool_name),
340                                 "dev%d_controlqueue_mpool",
341                                 dev->data->dev_id);
342         }
343         size = RTE_ALIGN_CEIL(sizeof(*vq) +
344                                 vq_size * sizeof(struct vq_desc_extra),
345                                 RTE_CACHE_LINE_SIZE);
346         vq = rte_zmalloc_socket(vq_name, size, RTE_CACHE_LINE_SIZE,
347                                 socket_id);
348         if (vq == NULL) {
349                 VIRTIO_CRYPTO_INIT_LOG_ERR("Can not allocate virtqueue");
350                 return -ENOMEM;
351         }
352
353         if (queue_type == VTCRYPTO_DATAQ) {
354                 /* pre-allocate a mempool and use it in the data plane to
355                  * improve performance
356                  */
357                 vq->mpool = rte_mempool_lookup(mpool_name);
358                 if (vq->mpool == NULL)
359                         vq->mpool = rte_mempool_create(mpool_name,
360                                         vq_size,
361                                         sizeof(struct virtio_crypto_op_cookie),
362                                         RTE_CACHE_LINE_SIZE, 0,
363                                         NULL, NULL, NULL, NULL, socket_id,
364                                         0);
365                 if (!vq->mpool) {
366                         VIRTIO_CRYPTO_DRV_LOG_ERR("Virtio Crypto PMD "
367                                         "Cannot create mempool");
368                         goto mpool_create_err;
369                 }
370                 for (i = 0; i < vq_size; i++) {
371                         vq->vq_descx[i].cookie =
372                                 rte_zmalloc("crypto PMD op cookie pointer",
373                                         sizeof(struct virtio_crypto_op_cookie),
374                                         RTE_CACHE_LINE_SIZE);
375                         if (vq->vq_descx[i].cookie == NULL) {
376                                 VIRTIO_CRYPTO_DRV_LOG_ERR("Failed to "
377                                                 "alloc mem for cookie");
378                                 goto cookie_alloc_err;
379                         }
380                 }
381         }
382
383         vq->hw = hw;
384         vq->dev_id = dev->data->dev_id;
385         vq->vq_queue_index = vtpci_queue_idx;
386         vq->vq_nentries = vq_size;
387
388         /*
389          * Using part of the vring entries is permitted, but the maximum
390          * is vq_size
391          */
392         if (nb_desc == 0 || nb_desc > vq_size)
393                 nb_desc = vq_size;
394         vq->vq_free_cnt = nb_desc;
395
396         /*
397          * Reserve a memzone for vring elements
398          */
399         size = vring_size(vq_size, VIRTIO_PCI_VRING_ALIGN);
400         vq->vq_ring_size = RTE_ALIGN_CEIL(size, VIRTIO_PCI_VRING_ALIGN);
401         VIRTIO_CRYPTO_INIT_LOG_DBG("%s vring_size: %d, rounded_vring_size: %d",
402                         (queue_type == VTCRYPTO_DATAQ) ? "dataq" : "ctrlq",
403                         size, vq->vq_ring_size);
404
405         mz = rte_memzone_reserve_aligned(vq_name, vq->vq_ring_size,
406                         socket_id, 0, VIRTIO_PCI_VRING_ALIGN);
407         if (mz == NULL) {
408                 if (rte_errno == EEXIST)
409                         mz = rte_memzone_lookup(vq_name);
410                 if (mz == NULL) {
411                         VIRTIO_CRYPTO_INIT_LOG_ERR("not enough memory");
412                         goto mz_reserve_err;
413                 }
414         }
415
416         /*
417          * Virtio PCI device VIRTIO_PCI_QUEUE_PF register is 32bit,
418          * and only accepts 32 bit page frame number.
419          * Check if the allocated physical memory exceeds 16TB.
420          */
421         if ((mz->phys_addr + vq->vq_ring_size - 1)
422                                 >> (VIRTIO_PCI_QUEUE_ADDR_SHIFT + 32)) {
423                 VIRTIO_CRYPTO_INIT_LOG_ERR("vring address shouldn't be "
424                                         "above 16TB!");
425                 goto vring_addr_err;
426         }
427
428         memset(mz->addr, 0, sizeof(mz->len));
429         vq->mz = mz;
430         vq->vq_ring_mem = mz->phys_addr;
431         vq->vq_ring_virt_mem = mz->addr;
432         VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_ring_mem(physical): 0x%"PRIx64,
433                                         (uint64_t)mz->phys_addr);
434         VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_ring_virt_mem: 0x%"PRIx64,
435                                         (uint64_t)(uintptr_t)mz->addr);
436
437         *pvq = vq;
438
439         return 0;
440
441 vring_addr_err:
442         rte_memzone_free(mz);
443 mz_reserve_err:
444 cookie_alloc_err:
445         rte_mempool_free(vq->mpool);
446         if (i != 0) {
447                 for (j = 0; j < i; j++)
448                         rte_free(vq->vq_descx[j].cookie);
449         }
450 mpool_create_err:
451         rte_free(vq);
452         return -ENOMEM;
453 }
454
455 static int
456 virtio_crypto_ctrlq_setup(struct rte_cryptodev *dev, uint16_t queue_idx)
457 {
458         int ret;
459         struct virtqueue *vq;
460         struct virtio_crypto_hw *hw = dev->data->dev_private;
461
462         /* if virtio device has started, do not touch the virtqueues */
463         if (dev->data->dev_started)
464                 return 0;
465
466         PMD_INIT_FUNC_TRACE();
467
468         ret = virtio_crypto_queue_setup(dev, VTCRYPTO_CTRLQ, queue_idx,
469                         0, SOCKET_ID_ANY, &vq);
470         if (ret < 0) {
471                 VIRTIO_CRYPTO_INIT_LOG_ERR("control vq initialization failed");
472                 return ret;
473         }
474
475         hw->cvq = vq;
476
477         return 0;
478 }
479
480 static void
481 virtio_crypto_free_queues(struct rte_cryptodev *dev)
482 {
483         unsigned int i;
484         struct virtio_crypto_hw *hw = dev->data->dev_private;
485
486         PMD_INIT_FUNC_TRACE();
487
488         /* control queue release */
489         virtio_crypto_queue_release(hw->cvq);
490
491         /* data queue release */
492         for (i = 0; i < hw->max_dataqueues; i++)
493                 virtio_crypto_queue_release(dev->data->queue_pairs[i]);
494 }
495
496 static int
497 virtio_crypto_dev_close(struct rte_cryptodev *dev __rte_unused)
498 {
499         return 0;
500 }
501
502 /*
503  * dev_ops for virtio, bare necessities for basic operation
504  */
505 static struct rte_cryptodev_ops virtio_crypto_dev_ops = {
506         /* Device related operations */
507         .dev_configure                   = virtio_crypto_dev_configure,
508         .dev_start                       = virtio_crypto_dev_start,
509         .dev_stop                        = virtio_crypto_dev_stop,
510         .dev_close                       = virtio_crypto_dev_close,
511         .dev_infos_get                   = virtio_crypto_dev_info_get,
512
513         .stats_get                       = virtio_crypto_dev_stats_get,
514         .stats_reset                     = virtio_crypto_dev_stats_reset,
515
516         .queue_pair_setup                = virtio_crypto_qp_setup,
517         .queue_pair_release              = virtio_crypto_qp_release,
518         .queue_pair_start                = NULL,
519         .queue_pair_stop                 = NULL,
520         .queue_pair_count                = NULL,
521
522         /* Crypto related operations */
523         .session_get_size       = virtio_crypto_sym_get_session_private_size,
524         .session_configure      = virtio_crypto_sym_configure_session,
525         .session_clear          = virtio_crypto_sym_clear_session,
526         .qp_attach_session = NULL,
527         .qp_detach_session = NULL
528 };
529
530 static void
531 virtio_crypto_update_stats(struct rte_cryptodev *dev,
532                 struct rte_cryptodev_stats *stats)
533 {
534         unsigned int i;
535         struct virtio_crypto_hw *hw = dev->data->dev_private;
536
537         PMD_INIT_FUNC_TRACE();
538
539         if (stats == NULL) {
540                 VIRTIO_CRYPTO_DRV_LOG_ERR("invalid pointer");
541                 return;
542         }
543
544         for (i = 0; i < hw->max_dataqueues; i++) {
545                 const struct virtqueue *data_queue
546                         = dev->data->queue_pairs[i];
547                 if (data_queue == NULL)
548                         continue;
549
550                 stats->enqueued_count += data_queue->packets_sent_total;
551                 stats->enqueue_err_count += data_queue->packets_sent_failed;
552
553                 stats->dequeued_count += data_queue->packets_received_total;
554                 stats->dequeue_err_count
555                         += data_queue->packets_received_failed;
556         }
557 }
558
559 static void
560 virtio_crypto_dev_stats_get(struct rte_cryptodev *dev,
561                 struct rte_cryptodev_stats *stats)
562 {
563         PMD_INIT_FUNC_TRACE();
564
565         virtio_crypto_update_stats(dev, stats);
566 }
567
568 static void
569 virtio_crypto_dev_stats_reset(struct rte_cryptodev *dev)
570 {
571         unsigned int i;
572         struct virtio_crypto_hw *hw = dev->data->dev_private;
573
574         PMD_INIT_FUNC_TRACE();
575
576         for (i = 0; i < hw->max_dataqueues; i++) {
577                 struct virtqueue *data_queue = dev->data->queue_pairs[i];
578                 if (data_queue == NULL)
579                         continue;
580
581                 data_queue->packets_sent_total = 0;
582                 data_queue->packets_sent_failed = 0;
583
584                 data_queue->packets_received_total = 0;
585                 data_queue->packets_received_failed = 0;
586         }
587 }
588
589 static int
590 virtio_crypto_qp_setup(struct rte_cryptodev *dev, uint16_t queue_pair_id,
591                 const struct rte_cryptodev_qp_conf *qp_conf,
592                 int socket_id,
593                 struct rte_mempool *session_pool __rte_unused)
594 {
595         int ret;
596         struct virtqueue *vq;
597
598         PMD_INIT_FUNC_TRACE();
599
600         /* if virtio dev is started, do not touch the virtqueues */
601         if (dev->data->dev_started)
602                 return 0;
603
604         ret = virtio_crypto_queue_setup(dev, VTCRYPTO_DATAQ, queue_pair_id,
605                         qp_conf->nb_descriptors, socket_id, &vq);
606         if (ret < 0) {
607                 VIRTIO_CRYPTO_INIT_LOG_ERR(
608                         "virtio crypto data queue initialization failed\n");
609                 return ret;
610         }
611
612         dev->data->queue_pairs[queue_pair_id] = vq;
613
614         return 0;
615 }
616
617 static int
618 virtio_crypto_qp_release(struct rte_cryptodev *dev, uint16_t queue_pair_id)
619 {
620         struct virtqueue *vq
621                 = (struct virtqueue *)dev->data->queue_pairs[queue_pair_id];
622
623         PMD_INIT_FUNC_TRACE();
624
625         if (vq == NULL) {
626                 VIRTIO_CRYPTO_DRV_LOG_DBG("vq already freed");
627                 return 0;
628         }
629
630         virtio_crypto_queue_release(vq);
631         return 0;
632 }
633
634 static int
635 virtio_negotiate_features(struct virtio_crypto_hw *hw, uint64_t req_features)
636 {
637         uint64_t host_features;
638
639         PMD_INIT_FUNC_TRACE();
640
641         /* Prepare guest_features: feature that driver wants to support */
642         VIRTIO_CRYPTO_INIT_LOG_DBG("guest_features before negotiate = %" PRIx64,
643                 req_features);
644
645         /* Read device(host) feature bits */
646         host_features = VTPCI_OPS(hw)->get_features(hw);
647         VIRTIO_CRYPTO_INIT_LOG_DBG("host_features before negotiate = %" PRIx64,
648                 host_features);
649
650         /*
651          * Negotiate features: Subset of device feature bits are written back
652          * guest feature bits.
653          */
654         hw->guest_features = req_features;
655         hw->guest_features = vtpci_cryptodev_negotiate_features(hw,
656                                                         host_features);
657         VIRTIO_CRYPTO_INIT_LOG_DBG("features after negotiate = %" PRIx64,
658                 hw->guest_features);
659
660         if (hw->modern) {
661                 if (!vtpci_with_feature(hw, VIRTIO_F_VERSION_1)) {
662                         VIRTIO_CRYPTO_INIT_LOG_ERR(
663                                 "VIRTIO_F_VERSION_1 features is not enabled.");
664                         return -1;
665                 }
666                 vtpci_cryptodev_set_status(hw,
667                         VIRTIO_CONFIG_STATUS_FEATURES_OK);
668                 if (!(vtpci_cryptodev_get_status(hw) &
669                         VIRTIO_CONFIG_STATUS_FEATURES_OK)) {
670                         VIRTIO_CRYPTO_INIT_LOG_ERR("failed to set FEATURES_OK "
671                                                 "status!");
672                         return -1;
673                 }
674         }
675
676         hw->req_guest_features = req_features;
677
678         return 0;
679 }
680
681 /* reset device and renegotiate features if needed */
682 static int
683 virtio_crypto_init_device(struct rte_cryptodev *cryptodev,
684         uint64_t req_features)
685 {
686         struct virtio_crypto_hw *hw = cryptodev->data->dev_private;
687         struct virtio_crypto_config local_config;
688         struct virtio_crypto_config *config = &local_config;
689
690         PMD_INIT_FUNC_TRACE();
691
692         /* Reset the device although not necessary at startup */
693         vtpci_cryptodev_reset(hw);
694
695         /* Tell the host we've noticed this device. */
696         vtpci_cryptodev_set_status(hw, VIRTIO_CONFIG_STATUS_ACK);
697
698         /* Tell the host we've known how to drive the device. */
699         vtpci_cryptodev_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER);
700         if (virtio_negotiate_features(hw, req_features) < 0)
701                 return -1;
702
703         /* Get status of the device */
704         vtpci_read_cryptodev_config(hw,
705                 offsetof(struct virtio_crypto_config, status),
706                 &config->status, sizeof(config->status));
707         if (config->status != VIRTIO_CRYPTO_S_HW_READY) {
708                 VIRTIO_CRYPTO_DRV_LOG_ERR("accelerator hardware is "
709                                 "not ready");
710                 return -1;
711         }
712
713         /* Get number of data queues */
714         vtpci_read_cryptodev_config(hw,
715                 offsetof(struct virtio_crypto_config, max_dataqueues),
716                 &config->max_dataqueues,
717                 sizeof(config->max_dataqueues));
718         hw->max_dataqueues = config->max_dataqueues;
719
720         VIRTIO_CRYPTO_INIT_LOG_DBG("hw->max_dataqueues=%d",
721                 hw->max_dataqueues);
722
723         return 0;
724 }
725
726 /*
727  * This function is based on probe() function
728  * It returns 0 on success.
729  */
730 static int
731 crypto_virtio_create(const char *name, struct rte_pci_device *pci_dev,
732                 struct rte_cryptodev_pmd_init_params *init_params)
733 {
734         struct rte_cryptodev *cryptodev;
735         struct virtio_crypto_hw *hw;
736
737         PMD_INIT_FUNC_TRACE();
738
739         cryptodev = rte_cryptodev_pmd_create(name, &pci_dev->device,
740                                         init_params);
741         if (cryptodev == NULL)
742                 return -ENODEV;
743
744         cryptodev->driver_id = cryptodev_virtio_driver_id;
745         cryptodev->dev_ops = &virtio_crypto_dev_ops;
746
747         cryptodev->enqueue_burst = virtio_crypto_pkt_tx_burst;
748         cryptodev->dequeue_burst = virtio_crypto_pkt_rx_burst;
749
750         cryptodev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
751                 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING;
752
753         hw = cryptodev->data->dev_private;
754         hw->dev_id = cryptodev->data->dev_id;
755         hw->virtio_dev_capabilities = virtio_capabilities;
756
757         VIRTIO_CRYPTO_INIT_LOG_DBG("dev %d vendorID=0x%x deviceID=0x%x",
758                 cryptodev->data->dev_id, pci_dev->id.vendor_id,
759                 pci_dev->id.device_id);
760
761         /* pci device init */
762         if (vtpci_cryptodev_init(pci_dev, hw))
763                 return -1;
764
765         if (virtio_crypto_init_device(cryptodev,
766                         VIRTIO_CRYPTO_PMD_GUEST_FEATURES) < 0)
767                 return -1;
768
769         return 0;
770 }
771
772 static int
773 virtio_crypto_dev_uninit(struct rte_cryptodev *cryptodev)
774 {
775         struct virtio_crypto_hw *hw = cryptodev->data->dev_private;
776
777         PMD_INIT_FUNC_TRACE();
778
779         if (rte_eal_process_type() == RTE_PROC_SECONDARY)
780                 return -EPERM;
781
782         if (cryptodev->data->dev_started) {
783                 virtio_crypto_dev_stop(cryptodev);
784                 virtio_crypto_dev_close(cryptodev);
785         }
786
787         cryptodev->dev_ops = NULL;
788         cryptodev->enqueue_burst = NULL;
789         cryptodev->dequeue_burst = NULL;
790
791         /* release control queue */
792         virtio_crypto_queue_release(hw->cvq);
793
794         rte_free(cryptodev->data);
795         cryptodev->data = NULL;
796
797         VIRTIO_CRYPTO_DRV_LOG_INFO("dev_uninit completed");
798
799         return 0;
800 }
801
802 static int
803 virtio_crypto_dev_configure(struct rte_cryptodev *cryptodev,
804         struct rte_cryptodev_config *config __rte_unused)
805 {
806         struct virtio_crypto_hw *hw = cryptodev->data->dev_private;
807
808         PMD_INIT_FUNC_TRACE();
809
810         if (virtio_crypto_init_device(cryptodev,
811                         VIRTIO_CRYPTO_PMD_GUEST_FEATURES) < 0)
812                 return -1;
813
814         /* setup control queue
815          * [0, 1, ... ,(config->max_dataqueues - 1)] are data queues
816          * config->max_dataqueues is the control queue
817          */
818         if (virtio_crypto_ctrlq_setup(cryptodev, hw->max_dataqueues) < 0) {
819                 VIRTIO_CRYPTO_INIT_LOG_ERR("control queue setup error");
820                 return -1;
821         }
822         virtio_crypto_ctrlq_start(cryptodev);
823
824         return 0;
825 }
826
827 static void
828 virtio_crypto_dev_stop(struct rte_cryptodev *dev)
829 {
830         struct virtio_crypto_hw *hw = dev->data->dev_private;
831
832         PMD_INIT_FUNC_TRACE();
833         VIRTIO_CRYPTO_DRV_LOG_DBG("virtio_dev_stop");
834
835         vtpci_cryptodev_reset(hw);
836
837         virtio_crypto_dev_free_mbufs(dev);
838         virtio_crypto_free_queues(dev);
839
840         dev->data->dev_started = 0;
841 }
842
843 static int
844 virtio_crypto_dev_start(struct rte_cryptodev *dev)
845 {
846         struct virtio_crypto_hw *hw = dev->data->dev_private;
847
848         if (dev->data->dev_started)
849                 return 0;
850
851         /* Do final configuration before queue engine starts */
852         virtio_crypto_dataq_start(dev);
853         vtpci_cryptodev_reinit_complete(hw);
854
855         dev->data->dev_started = 1;
856
857         return 0;
858 }
859
860 static void
861 virtio_crypto_dev_free_mbufs(struct rte_cryptodev *dev)
862 {
863         uint32_t i;
864         struct virtio_crypto_hw *hw = dev->data->dev_private;
865
866         for (i = 0; i < hw->max_dataqueues; i++) {
867                 VIRTIO_CRYPTO_INIT_LOG_DBG("Before freeing dataq[%d] used "
868                         "and unused buf", i);
869                 VIRTQUEUE_DUMP((struct virtqueue *)
870                         dev->data->queue_pairs[i]);
871
872                 VIRTIO_CRYPTO_INIT_LOG_DBG("queue_pairs[%d]=%p",
873                                 i, dev->data->queue_pairs[i]);
874
875                 virtqueue_detatch_unused(dev->data->queue_pairs[i]);
876
877                 VIRTIO_CRYPTO_INIT_LOG_DBG("After freeing dataq[%d] used and "
878                                         "unused buf", i);
879                 VIRTQUEUE_DUMP(
880                         (struct virtqueue *)dev->data->queue_pairs[i]);
881         }
882 }
883
884 static unsigned int
885 virtio_crypto_sym_get_session_private_size(
886                 struct rte_cryptodev *dev __rte_unused)
887 {
888         PMD_INIT_FUNC_TRACE();
889
890         return RTE_ALIGN_CEIL(sizeof(struct virtio_crypto_session), 16);
891 }
892
893 static int
894 virtio_crypto_check_sym_session_paras(
895                 struct rte_cryptodev *dev)
896 {
897         struct virtio_crypto_hw *hw;
898
899         PMD_INIT_FUNC_TRACE();
900
901         if (unlikely(dev == NULL)) {
902                 VIRTIO_CRYPTO_SESSION_LOG_ERR("dev is NULL");
903                 return -1;
904         }
905         if (unlikely(dev->data == NULL)) {
906                 VIRTIO_CRYPTO_SESSION_LOG_ERR("dev->data is NULL");
907                 return -1;
908         }
909         hw = dev->data->dev_private;
910         if (unlikely(hw == NULL)) {
911                 VIRTIO_CRYPTO_SESSION_LOG_ERR("hw is NULL");
912                 return -1;
913         }
914         if (unlikely(hw->cvq == NULL)) {
915                 VIRTIO_CRYPTO_SESSION_LOG_ERR("vq is NULL");
916                 return -1;
917         }
918
919         return 0;
920 }
921
922 static int
923 virtio_crypto_check_sym_clear_session_paras(
924                 struct rte_cryptodev *dev,
925                 struct rte_cryptodev_sym_session *sess)
926 {
927         PMD_INIT_FUNC_TRACE();
928
929         if (sess == NULL) {
930                 VIRTIO_CRYPTO_SESSION_LOG_ERR("sym_session is NULL");
931                 return -1;
932         }
933
934         return virtio_crypto_check_sym_session_paras(dev);
935 }
936
937 #define NUM_ENTRY_SYM_CLEAR_SESSION 2
938
939 static void
940 virtio_crypto_sym_clear_session(
941                 struct rte_cryptodev *dev,
942                 struct rte_cryptodev_sym_session *sess)
943 {
944         struct virtio_crypto_hw *hw;
945         struct virtqueue *vq;
946         struct virtio_crypto_session *session;
947         struct virtio_crypto_op_ctrl_req *ctrl;
948         struct vring_desc *desc;
949         uint8_t *status;
950         uint8_t needed = 1;
951         uint32_t head;
952         uint8_t *malloc_virt_addr;
953         uint64_t malloc_phys_addr;
954         uint8_t len_inhdr = sizeof(struct virtio_crypto_inhdr);
955         uint32_t len_op_ctrl_req = sizeof(struct virtio_crypto_op_ctrl_req);
956         uint32_t desc_offset = len_op_ctrl_req + len_inhdr;
957
958         PMD_INIT_FUNC_TRACE();
959
960         if (virtio_crypto_check_sym_clear_session_paras(dev, sess) < 0)
961                 return;
962
963         hw = dev->data->dev_private;
964         vq = hw->cvq;
965         session = (struct virtio_crypto_session *)get_session_private_data(
966                 sess, cryptodev_virtio_driver_id);
967         if (session == NULL) {
968                 VIRTIO_CRYPTO_SESSION_LOG_ERR("Invalid session parameter");
969                 return;
970         }
971
972         VIRTIO_CRYPTO_SESSION_LOG_INFO("vq->vq_desc_head_idx = %d, "
973                         "vq = %p", vq->vq_desc_head_idx, vq);
974
975         if (vq->vq_free_cnt < needed) {
976                 VIRTIO_CRYPTO_SESSION_LOG_ERR(
977                                 "vq->vq_free_cnt = %d is less than %d, "
978                                 "not enough", vq->vq_free_cnt, needed);
979                 return;
980         }
981
982         /*
983          * malloc memory to store information of ctrl request op,
984          * returned status and desc vring
985          */
986         malloc_virt_addr = rte_malloc(NULL, len_op_ctrl_req + len_inhdr
987                 + NUM_ENTRY_SYM_CLEAR_SESSION
988                 * sizeof(struct vring_desc), RTE_CACHE_LINE_SIZE);
989         if (malloc_virt_addr == NULL) {
990                 VIRTIO_CRYPTO_SESSION_LOG_ERR("not enough heap room");
991                 return;
992         }
993         malloc_phys_addr = rte_malloc_virt2iova(malloc_virt_addr);
994
995         /* assign ctrl request op part */
996         ctrl = (struct virtio_crypto_op_ctrl_req *)malloc_virt_addr;
997         ctrl->header.opcode = VIRTIO_CRYPTO_CIPHER_DESTROY_SESSION;
998         /* default data virtqueue is 0 */
999         ctrl->header.queue_id = 0;
1000         ctrl->u.destroy_session.session_id = session->session_id;
1001
1002         /* status part */
1003         status = &(((struct virtio_crypto_inhdr *)
1004                 ((uint8_t *)malloc_virt_addr + len_op_ctrl_req))->status);
1005         *status = VIRTIO_CRYPTO_ERR;
1006
1007         /* indirect desc vring part */
1008         desc = (struct vring_desc *)((uint8_t *)malloc_virt_addr
1009                 + desc_offset);
1010
1011         /* ctrl request part */
1012         desc[0].addr = malloc_phys_addr;
1013         desc[0].len = len_op_ctrl_req;
1014         desc[0].flags = VRING_DESC_F_NEXT;
1015         desc[0].next = 1;
1016
1017         /* status part */
1018         desc[1].addr = malloc_phys_addr + len_op_ctrl_req;
1019         desc[1].len = len_inhdr;
1020         desc[1].flags = VRING_DESC_F_WRITE;
1021
1022         /* use only a single desc entry */
1023         head = vq->vq_desc_head_idx;
1024         vq->vq_ring.desc[head].flags = VRING_DESC_F_INDIRECT;
1025         vq->vq_ring.desc[head].addr = malloc_phys_addr + desc_offset;
1026         vq->vq_ring.desc[head].len
1027                 = NUM_ENTRY_SYM_CLEAR_SESSION
1028                 * sizeof(struct vring_desc);
1029         vq->vq_free_cnt -= needed;
1030
1031         vq->vq_desc_head_idx = vq->vq_ring.desc[head].next;
1032
1033         vq_update_avail_ring(vq, head);
1034         vq_update_avail_idx(vq);
1035
1036         VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_queue_index = %d",
1037                                         vq->vq_queue_index);
1038
1039         virtqueue_notify(vq);
1040
1041         rte_rmb();
1042         while (vq->vq_used_cons_idx == vq->vq_ring.used->idx) {
1043                 rte_rmb();
1044                 usleep(100);
1045         }
1046
1047         while (vq->vq_used_cons_idx != vq->vq_ring.used->idx) {
1048                 uint32_t idx, desc_idx, used_idx;
1049                 struct vring_used_elem *uep;
1050
1051                 used_idx = (uint32_t)(vq->vq_used_cons_idx
1052                                 & (vq->vq_nentries - 1));
1053                 uep = &vq->vq_ring.used->ring[used_idx];
1054                 idx = (uint32_t) uep->id;
1055                 desc_idx = idx;
1056                 while (vq->vq_ring.desc[desc_idx].flags
1057                                 & VRING_DESC_F_NEXT) {
1058                         desc_idx = vq->vq_ring.desc[desc_idx].next;
1059                         vq->vq_free_cnt++;
1060                 }
1061
1062                 vq->vq_ring.desc[desc_idx].next = vq->vq_desc_head_idx;
1063                 vq->vq_desc_head_idx = idx;
1064                 vq->vq_used_cons_idx++;
1065                 vq->vq_free_cnt++;
1066         }
1067
1068         if (*status != VIRTIO_CRYPTO_OK) {
1069                 VIRTIO_CRYPTO_SESSION_LOG_ERR("Close session failed "
1070                                 "status=%"PRIu32", session_id=%"PRIu64"",
1071                                 *status, session->session_id);
1072                 rte_free(malloc_virt_addr);
1073                 return;
1074         }
1075
1076         VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_free_cnt=%d\n"
1077                         "vq->vq_desc_head_idx=%d",
1078                         vq->vq_free_cnt, vq->vq_desc_head_idx);
1079
1080         VIRTIO_CRYPTO_SESSION_LOG_INFO("Close session %"PRIu64" successfully ",
1081                         session->session_id);
1082
1083         memset(sess, 0, sizeof(struct virtio_crypto_session));
1084         rte_free(malloc_virt_addr);
1085 }
1086
1087 static struct rte_crypto_cipher_xform *
1088 virtio_crypto_get_cipher_xform(struct rte_crypto_sym_xform *xform)
1089 {
1090         do {
1091                 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
1092                         return &xform->cipher;
1093
1094                 xform = xform->next;
1095         } while (xform);
1096
1097         return NULL;
1098 }
1099
1100 static struct rte_crypto_auth_xform *
1101 virtio_crypto_get_auth_xform(struct rte_crypto_sym_xform *xform)
1102 {
1103         do {
1104                 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH)
1105                         return &xform->auth;
1106
1107                 xform = xform->next;
1108         } while (xform);
1109
1110         return NULL;
1111 }
1112
1113 /** Get xform chain order */
1114 static int
1115 virtio_crypto_get_chain_order(struct rte_crypto_sym_xform *xform)
1116 {
1117         if (xform == NULL)
1118                 return -1;
1119
1120         /* Cipher Only */
1121         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
1122                         xform->next == NULL)
1123                 return VIRTIO_CRYPTO_CMD_CIPHER;
1124
1125         /* Authentication Only */
1126         if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
1127                         xform->next == NULL)
1128                 return VIRTIO_CRYPTO_CMD_AUTH;
1129
1130         /* Authenticate then Cipher */
1131         if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
1132                         xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
1133                 return VIRTIO_CRYPTO_CMD_HASH_CIPHER;
1134
1135         /* Cipher then Authenticate */
1136         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
1137                         xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
1138                 return VIRTIO_CRYPTO_CMD_CIPHER_HASH;
1139
1140         return -1;
1141 }
1142
1143 static int
1144 virtio_crypto_sym_pad_cipher_param(
1145                 struct virtio_crypto_cipher_session_para *para,
1146                 struct rte_crypto_cipher_xform *cipher_xform)
1147 {
1148         switch (cipher_xform->algo) {
1149         case RTE_CRYPTO_CIPHER_AES_CBC:
1150                 para->algo = VIRTIO_CRYPTO_CIPHER_AES_CBC;
1151                 break;
1152         default:
1153                 VIRTIO_CRYPTO_SESSION_LOG_ERR("Crypto: Unsupported "
1154                                 "Cipher alg %u", cipher_xform->algo);
1155                 return -1;
1156         }
1157
1158         para->keylen = cipher_xform->key.length;
1159         switch (cipher_xform->op) {
1160         case RTE_CRYPTO_CIPHER_OP_ENCRYPT:
1161                 para->op = VIRTIO_CRYPTO_OP_ENCRYPT;
1162                 break;
1163         case RTE_CRYPTO_CIPHER_OP_DECRYPT:
1164                 para->op = VIRTIO_CRYPTO_OP_DECRYPT;
1165                 break;
1166         default:
1167                 VIRTIO_CRYPTO_SESSION_LOG_ERR("Unsupported cipher operation "
1168                                         "parameter");
1169                 return -1;
1170         }
1171
1172         return 0;
1173 }
1174
1175 static int
1176 virtio_crypto_sym_pad_auth_param(
1177                 struct virtio_crypto_op_ctrl_req *ctrl,
1178                 struct rte_crypto_auth_xform *auth_xform)
1179 {
1180         uint32_t *algo;
1181         struct virtio_crypto_alg_chain_session_para *para =
1182                 &(ctrl->u.sym_create_session.u.chain.para);
1183
1184         switch (ctrl->u.sym_create_session.u.chain.para.hash_mode) {
1185         case VIRTIO_CRYPTO_SYM_HASH_MODE_PLAIN:
1186                 algo = &(para->u.hash_param.algo);
1187                 break;
1188         case VIRTIO_CRYPTO_SYM_HASH_MODE_AUTH:
1189                 algo = &(para->u.mac_param.algo);
1190                 break;
1191         default:
1192                 VIRTIO_CRYPTO_SESSION_LOG_ERR("Unsupported hash mode %u "
1193                         "specified",
1194                         ctrl->u.sym_create_session.u.chain.para.hash_mode);
1195                 return -1;
1196         }
1197
1198         switch (auth_xform->algo) {
1199         case RTE_CRYPTO_AUTH_SHA1_HMAC:
1200                 *algo = VIRTIO_CRYPTO_MAC_HMAC_SHA1;
1201                 break;
1202         default:
1203                 VIRTIO_CRYPTO_SESSION_LOG_ERR(
1204                         "Crypto: Undefined Hash algo %u specified",
1205                         auth_xform->algo);
1206                 return -1;
1207         }
1208
1209         return 0;
1210 }
1211
1212 static int
1213 virtio_crypto_sym_pad_op_ctrl_req(
1214                 struct virtio_crypto_op_ctrl_req *ctrl,
1215                 struct rte_crypto_sym_xform *xform, bool is_chainned,
1216                 uint8_t **cipher_key_data, uint8_t **auth_key_data,
1217                 struct virtio_crypto_session *session)
1218 {
1219         int ret;
1220         struct rte_crypto_auth_xform *auth_xform = NULL;
1221         struct rte_crypto_cipher_xform *cipher_xform = NULL;
1222
1223         /* Get cipher xform from crypto xform chain */
1224         cipher_xform = virtio_crypto_get_cipher_xform(xform);
1225         if (cipher_xform) {
1226                 if (is_chainned)
1227                         ret = virtio_crypto_sym_pad_cipher_param(
1228                                 &ctrl->u.sym_create_session.u.chain.para
1229                                                 .cipher_param, cipher_xform);
1230                 else
1231                         ret = virtio_crypto_sym_pad_cipher_param(
1232                                 &ctrl->u.sym_create_session.u.cipher.para,
1233                                 cipher_xform);
1234
1235                 if (ret < 0) {
1236                         VIRTIO_CRYPTO_SESSION_LOG_ERR(
1237                                 "pad cipher parameter failed");
1238                         return -1;
1239                 }
1240
1241                 *cipher_key_data = cipher_xform->key.data;
1242
1243                 session->iv.offset = cipher_xform->iv.offset;
1244                 session->iv.length = cipher_xform->iv.length;
1245         }
1246
1247         /* Get auth xform from crypto xform chain */
1248         auth_xform = virtio_crypto_get_auth_xform(xform);
1249         if (auth_xform) {
1250                 /* FIXME: support VIRTIO_CRYPTO_SYM_HASH_MODE_NESTED */
1251                 struct virtio_crypto_alg_chain_session_para *para =
1252                         &(ctrl->u.sym_create_session.u.chain.para);
1253                 if (auth_xform->key.length) {
1254                         para->hash_mode = VIRTIO_CRYPTO_SYM_HASH_MODE_AUTH;
1255                         para->u.mac_param.auth_key_len =
1256                                 (uint32_t)auth_xform->key.length;
1257                         para->u.mac_param.hash_result_len =
1258                                 auth_xform->digest_length;
1259
1260                         *auth_key_data = auth_xform->key.data;
1261                 } else {
1262                         para->hash_mode = VIRTIO_CRYPTO_SYM_HASH_MODE_PLAIN;
1263                         para->u.hash_param.hash_result_len =
1264                                 auth_xform->digest_length;
1265                 }
1266
1267                 ret = virtio_crypto_sym_pad_auth_param(ctrl, auth_xform);
1268                 if (ret < 0) {
1269                         VIRTIO_CRYPTO_SESSION_LOG_ERR("pad auth parameter "
1270                                                 "failed");
1271                         return -1;
1272                 }
1273         }
1274
1275         return 0;
1276 }
1277
1278 static int
1279 virtio_crypto_check_sym_configure_session_paras(
1280                 struct rte_cryptodev *dev,
1281                 struct rte_crypto_sym_xform *xform,
1282                 struct rte_cryptodev_sym_session *sym_sess,
1283                 struct rte_mempool *mempool)
1284 {
1285         if (unlikely(xform == NULL) || unlikely(sym_sess == NULL) ||
1286                 unlikely(mempool == NULL)) {
1287                 VIRTIO_CRYPTO_SESSION_LOG_ERR("NULL pointer");
1288                 return -1;
1289         }
1290
1291         if (virtio_crypto_check_sym_session_paras(dev) < 0)
1292                 return -1;
1293
1294         return 0;
1295 }
1296
1297 static int
1298 virtio_crypto_sym_configure_session(
1299                 struct rte_cryptodev *dev,
1300                 struct rte_crypto_sym_xform *xform,
1301                 struct rte_cryptodev_sym_session *sess,
1302                 struct rte_mempool *mempool)
1303 {
1304         int ret;
1305         struct virtio_crypto_session crypto_sess;
1306         void *session_private = &crypto_sess;
1307         struct virtio_crypto_session *session;
1308         struct virtio_crypto_op_ctrl_req *ctrl_req;
1309         enum virtio_crypto_cmd_id cmd_id;
1310         uint8_t *cipher_key_data = NULL;
1311         uint8_t *auth_key_data = NULL;
1312         struct virtio_crypto_hw *hw;
1313         struct virtqueue *control_vq;
1314
1315         PMD_INIT_FUNC_TRACE();
1316
1317         ret = virtio_crypto_check_sym_configure_session_paras(dev, xform,
1318                         sess, mempool);
1319         if (ret < 0) {
1320                 VIRTIO_CRYPTO_SESSION_LOG_ERR("Invalid parameters");
1321                 return ret;
1322         }
1323
1324         if (rte_mempool_get(mempool, &session_private)) {
1325                 VIRTIO_CRYPTO_SESSION_LOG_ERR(
1326                         "Couldn't get object from session mempool");
1327                 return -ENOMEM;
1328         }
1329
1330         session = (struct virtio_crypto_session *)session_private;
1331         memset(session, 0, sizeof(struct virtio_crypto_session));
1332         ctrl_req = &session->ctrl;
1333         ctrl_req->header.opcode = VIRTIO_CRYPTO_CIPHER_CREATE_SESSION;
1334         /* FIXME: support multiqueue */
1335         ctrl_req->header.queue_id = 0;
1336
1337         hw = dev->data->dev_private;
1338         control_vq = hw->cvq;
1339
1340         cmd_id = virtio_crypto_get_chain_order(xform);
1341         if (cmd_id == VIRTIO_CRYPTO_CMD_CIPHER_HASH)
1342                 ctrl_req->u.sym_create_session.u.chain.para.alg_chain_order
1343                         = VIRTIO_CRYPTO_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH;
1344         if (cmd_id == VIRTIO_CRYPTO_CMD_HASH_CIPHER)
1345                 ctrl_req->u.sym_create_session.u.chain.para.alg_chain_order
1346                         = VIRTIO_CRYPTO_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER;
1347
1348         switch (cmd_id) {
1349         case VIRTIO_CRYPTO_CMD_CIPHER_HASH:
1350         case VIRTIO_CRYPTO_CMD_HASH_CIPHER:
1351                 ctrl_req->u.sym_create_session.op_type
1352                         = VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING;
1353
1354                 ret = virtio_crypto_sym_pad_op_ctrl_req(ctrl_req,
1355                         xform, true, &cipher_key_data, &auth_key_data, session);
1356                 if (ret < 0) {
1357                         VIRTIO_CRYPTO_SESSION_LOG_ERR(
1358                                 "padding sym op ctrl req failed");
1359                         goto error_out;
1360                 }
1361                 ret = virtio_crypto_send_command(control_vq, ctrl_req,
1362                         cipher_key_data, auth_key_data, session);
1363                 if (ret < 0) {
1364                         VIRTIO_CRYPTO_SESSION_LOG_ERR(
1365                                 "create session failed: %d", ret);
1366                         goto error_out;
1367                 }
1368                 break;
1369         case VIRTIO_CRYPTO_CMD_CIPHER:
1370                 ctrl_req->u.sym_create_session.op_type
1371                         = VIRTIO_CRYPTO_SYM_OP_CIPHER;
1372                 ret = virtio_crypto_sym_pad_op_ctrl_req(ctrl_req, xform,
1373                         false, &cipher_key_data, &auth_key_data, session);
1374                 if (ret < 0) {
1375                         VIRTIO_CRYPTO_SESSION_LOG_ERR(
1376                                 "padding sym op ctrl req failed");
1377                         goto error_out;
1378                 }
1379                 ret = virtio_crypto_send_command(control_vq, ctrl_req,
1380                         cipher_key_data, NULL, session);
1381                 if (ret < 0) {
1382                         VIRTIO_CRYPTO_SESSION_LOG_ERR(
1383                                 "create session failed: %d", ret);
1384                         goto error_out;
1385                 }
1386                 break;
1387         default:
1388                 VIRTIO_CRYPTO_SESSION_LOG_ERR(
1389                         "Unsupported operation chain order parameter");
1390                 goto error_out;
1391         }
1392
1393         set_session_private_data(sess, dev->driver_id,
1394                 session_private);
1395
1396         return 0;
1397
1398 error_out:
1399         return -1;
1400 }
1401
1402 static void
1403 virtio_crypto_dev_info_get(struct rte_cryptodev *dev,
1404                 struct rte_cryptodev_info *info)
1405 {
1406         struct virtio_crypto_hw *hw = dev->data->dev_private;
1407
1408         PMD_INIT_FUNC_TRACE();
1409
1410         if (info != NULL) {
1411                 info->driver_id = cryptodev_virtio_driver_id;
1412                 info->pci_dev = RTE_DEV_TO_PCI(dev->device);
1413                 info->feature_flags = dev->feature_flags;
1414                 info->max_nb_queue_pairs = hw->max_dataqueues;
1415                 info->sym.max_nb_sessions =
1416                         RTE_VIRTIO_CRYPTO_PMD_MAX_NB_SESSIONS;
1417                 info->capabilities = hw->virtio_dev_capabilities;
1418         }
1419 }
1420
1421 static int
1422 crypto_virtio_pci_probe(
1423         struct rte_pci_driver *pci_drv __rte_unused,
1424         struct rte_pci_device *pci_dev)
1425 {
1426         struct rte_cryptodev_pmd_init_params init_params = {
1427                 .name = "",
1428                 .socket_id = rte_socket_id(),
1429                 .private_data_size = sizeof(struct virtio_crypto_hw),
1430                 .max_nb_sessions = RTE_VIRTIO_CRYPTO_PMD_MAX_NB_SESSIONS
1431         };
1432         char name[RTE_CRYPTODEV_NAME_MAX_LEN];
1433
1434         VIRTIO_CRYPTO_DRV_LOG_DBG("Found Crypto device at %02x:%02x.%x",
1435                         pci_dev->addr.bus,
1436                         pci_dev->addr.devid,
1437                         pci_dev->addr.function);
1438
1439         rte_pci_device_name(&pci_dev->addr, name, sizeof(name));
1440
1441         return crypto_virtio_create(name, pci_dev, &init_params);
1442 }
1443
1444 static int
1445 crypto_virtio_pci_remove(
1446         struct rte_pci_device *pci_dev __rte_unused)
1447 {
1448         struct rte_cryptodev *cryptodev;
1449         char cryptodev_name[RTE_CRYPTODEV_NAME_MAX_LEN];
1450
1451         if (pci_dev == NULL)
1452                 return -EINVAL;
1453
1454         rte_pci_device_name(&pci_dev->addr, cryptodev_name,
1455                         sizeof(cryptodev_name));
1456
1457         cryptodev = rte_cryptodev_pmd_get_named_dev(cryptodev_name);
1458         if (cryptodev == NULL)
1459                 return -ENODEV;
1460
1461         return virtio_crypto_dev_uninit(cryptodev);
1462 }
1463
1464 static struct rte_pci_driver rte_virtio_crypto_driver = {
1465         .id_table = pci_id_virtio_crypto_map,
1466         .drv_flags = 0,
1467         .probe = crypto_virtio_pci_probe,
1468         .remove = crypto_virtio_pci_remove
1469 };
1470
1471 static struct cryptodev_driver virtio_crypto_drv;
1472
1473 RTE_PMD_REGISTER_PCI(CRYPTODEV_NAME_VIRTIO_PMD, rte_virtio_crypto_driver);
1474 RTE_PMD_REGISTER_CRYPTO_DRIVER(virtio_crypto_drv,
1475         rte_virtio_crypto_driver.driver,
1476         cryptodev_virtio_driver_id);
1477
1478 RTE_INIT(virtio_crypto_init_log);
1479 static void
1480 virtio_crypto_init_log(void)
1481 {
1482         virtio_crypto_logtype_init = rte_log_register("pmd.crypto.virtio.init");
1483         if (virtio_crypto_logtype_init >= 0)
1484                 rte_log_set_level(virtio_crypto_logtype_init, RTE_LOG_NOTICE);
1485
1486         virtio_crypto_logtype_session =
1487                 rte_log_register("pmd.crypto.virtio.session");
1488         if (virtio_crypto_logtype_session >= 0)
1489                 rte_log_set_level(virtio_crypto_logtype_session,
1490                                 RTE_LOG_NOTICE);
1491
1492         virtio_crypto_logtype_rx = rte_log_register("pmd.crypto.virtio.rx");
1493         if (virtio_crypto_logtype_rx >= 0)
1494                 rte_log_set_level(virtio_crypto_logtype_rx, RTE_LOG_NOTICE);
1495
1496         virtio_crypto_logtype_tx = rte_log_register("pmd.crypto.virtio.tx");
1497         if (virtio_crypto_logtype_tx >= 0)
1498                 rte_log_set_level(virtio_crypto_logtype_tx, RTE_LOG_NOTICE);
1499
1500         virtio_crypto_logtype_driver =
1501                 rte_log_register("pmd.crypto.virtio.driver");
1502         if (virtio_crypto_logtype_driver >= 0)
1503                 rte_log_set_level(virtio_crypto_logtype_driver, RTE_LOG_NOTICE);
1504 }