drivers/regex/cn9k/cn9k_regexdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright (C) 2020 Marvell International Ltd.
   3  */
   4
   5 #include <stdio.h>
   6 #include <unistd.h>
   7
   8 #include <rte_malloc.h>
   9 #include <rte_memzone.h>
  10 #include <rte_regexdev.h>
  11 #include <rte_regexdev_core.h>
  12 #include <rte_regexdev_driver.h>
  13
  14
  15 /* REE common headers */
  16 #include "cn9k_regexdev.h"
  17 #include "cn9k_regexdev_compiler.h"
  18
  19
  20 /* HW matches are at offset 0x80 from RES_PTR_ADDR
  21  * In op structure matches starts at W5 (0x28)
  22  * There is a need to copy to 0x28 to 0x80 The matches that are at the tail
  23  * Which are 88 B. Each match holds 8 B, so up to 11 matches can be copied
  24  */
  25 #define REE_NUM_MATCHES_ALIGN   11
  26 /* The REE co-processor will write up to 254 job match structures
  27  * (REE_MATCH_S) starting at address [RES_PTR_ADDR] + 0x80.
  28  */
  29 #define REE_MATCH_OFFSET        0x80
  30
  31 #define REE_MAX_RULES_PER_GROUP 0xFFFF
  32 #define REE_MAX_GROUPS 0xFFFF
  33
  34
  35 #define REE_RULE_DB_VERSION     2
  36 #define REE_RULE_DB_REVISION    0
  37
  38 struct ree_rule_db_entry {
  39         uint8_t         type;
  40         uint32_t        addr;
  41         uint64_t        value;
  42 };
  43
  44 struct ree_rule_db {
  45         uint32_t version;
  46         uint32_t revision;
  47         uint32_t number_of_entries;
  48         struct ree_rule_db_entry entries[];
  49 } __rte_packed;
  50
  51 static void
  52 qp_memzone_name_get(char *name, int size, int dev_id, int qp_id)
  53 {
  54         snprintf(name, size, "cn9k_ree_lf_mem_%u:%u", dev_id, qp_id);
  55 }
  56
  57 static struct roc_ree_qp *
  58 ree_qp_create(const struct rte_regexdev *dev, uint16_t qp_id)
  59 {
  60         struct cn9k_ree_data *data = dev->data->dev_private;
  61         uint64_t pg_sz = sysconf(_SC_PAGESIZE);
  62         struct roc_ree_vf *vf = &data->vf;
  63         const struct rte_memzone *lf_mem;
  64         uint32_t len, iq_len, size_div2;
  65         char name[RTE_MEMZONE_NAMESIZE];
  66         uint64_t used_len, iova;
  67         struct roc_ree_qp *qp;
  68         uint8_t *va;
  69         int ret;
  70
  71         /* Allocate queue pair */
  72         qp = rte_zmalloc("CN9K Regex PMD Queue Pair", sizeof(*qp),
  73                                 ROC_ALIGN);
  74         if (qp == NULL) {
  75                 cn9k_err("Could not allocate queue pair");
  76                 return NULL;
  77         }
  78
  79         iq_len = REE_IQ_LEN;
  80
  81         /*
  82          * Queue size must be in units of 128B 2 * REE_INST_S (which is 64B),
  83          * and a power of 2.
  84          * effective queue size to software is (size - 1) * 128
  85          */
  86         size_div2 = iq_len >> 1;
  87
  88         /* For pending queue */
  89         len = iq_len * RTE_ALIGN(sizeof(struct roc_ree_rid), 8);
  90
  91         /* So that instruction queues start as pg size aligned */
  92         len = RTE_ALIGN(len, pg_sz);
  93
  94         /* For instruction queues */
  95         len += REE_IQ_LEN * sizeof(union roc_ree_inst);
  96
  97         /* Waste after instruction queues */
  98         len = RTE_ALIGN(len, pg_sz);
  99
 100         qp_memzone_name_get(name, RTE_MEMZONE_NAMESIZE, dev->data->dev_id,
 101                             qp_id);
 102
 103         lf_mem = rte_memzone_reserve_aligned(name, len, rte_socket_id(),
 104                         RTE_MEMZONE_SIZE_HINT_ONLY | RTE_MEMZONE_256MB,
 105                         RTE_CACHE_LINE_SIZE);
 106         if (lf_mem == NULL) {
 107                 cn9k_err("Could not allocate reserved memzone");
 108                 goto qp_free;
 109         }
 110
 111         va = lf_mem->addr;
 112         iova = lf_mem->iova;
 113
 114         memset(va, 0, len);
 115
 116         /* Initialize pending queue */
 117         qp->pend_q.rid_queue = (struct roc_ree_rid *)va;
 118         qp->pend_q.enq_tail = 0;
 119         qp->pend_q.deq_head = 0;
 120         qp->pend_q.pending_count = 0;
 121
 122         used_len = iq_len * RTE_ALIGN(sizeof(struct roc_ree_rid), 8);
 123         used_len = RTE_ALIGN(used_len, pg_sz);
 124         iova += used_len;
 125
 126         qp->iq_dma_addr = iova;
 127         qp->id = qp_id;
 128         qp->base = roc_ree_qp_get_base(vf, qp_id);
 129         qp->roc_regexdev_jobid = 0;
 130         qp->write_offset = 0;
 131
 132         ret = roc_ree_iq_enable(vf, qp, REE_QUEUE_HI_PRIO, size_div2);
 133         if (ret) {
 134                 cn9k_err("Could not enable instruction queue");
 135                 goto qp_free;
 136         }
 137
 138         return qp;
 139
 140 qp_free:
 141         rte_free(qp);
 142         return NULL;
 143 }
 144
 145 static int
 146 ree_qp_destroy(const struct rte_regexdev *dev, struct roc_ree_qp *qp)
 147 {
 148         const struct rte_memzone *lf_mem;
 149         char name[RTE_MEMZONE_NAMESIZE];
 150         int ret;
 151
 152         roc_ree_iq_disable(qp);
 153
 154         qp_memzone_name_get(name, RTE_MEMZONE_NAMESIZE, dev->data->dev_id,
 155                             qp->id);
 156
 157         lf_mem = rte_memzone_lookup(name);
 158
 159         ret = rte_memzone_free(lf_mem);
 160         if (ret)
 161                 return ret;
 162
 163         rte_free(qp);
 164
 165         return 0;
 166 }
 167
 168 static int
 169 ree_queue_pair_release(struct rte_regexdev *dev, uint16_t qp_id)
 170 {
 171         struct cn9k_ree_data *data = dev->data->dev_private;
 172         struct roc_ree_qp *qp = data->queue_pairs[qp_id];
 173         int ret;
 174
 175         ree_func_trace("Queue=%d", qp_id);
 176
 177         if (qp == NULL)
 178                 return -EINVAL;
 179
 180         ret = ree_qp_destroy(dev, qp);
 181         if (ret) {
 182                 cn9k_err("Could not destroy queue pair %d", qp_id);
 183                 return ret;
 184         }
 185
 186         data->queue_pairs[qp_id] = NULL;
 187
 188         return 0;
 189 }
 190
 191 static struct rte_regexdev *
 192 ree_dev_register(const char *name)
 193 {
 194         struct rte_regexdev *dev;
 195
 196         cn9k_ree_dbg("Creating regexdev %s\n", name);
 197
 198         /* allocate device structure */
 199         dev = rte_regexdev_register(name);
 200         if (dev == NULL) {
 201                 cn9k_err("Failed to allocate regex device for %s", name);
 202                 return NULL;
 203         }
 204
 205         /* allocate private device structure */
 206         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 207                 dev->data->dev_private =
 208                                 rte_zmalloc_socket("regexdev device private",
 209                                                 sizeof(struct cn9k_ree_data),
 210                                                 RTE_CACHE_LINE_SIZE,
 211                                                 rte_socket_id());
 212
 213                 if (dev->data->dev_private == NULL) {
 214                         cn9k_err("Cannot allocate memory for dev %s private data",
 215                                         name);
 216
 217                         rte_regexdev_unregister(dev);
 218                         return NULL;
 219                 }
 220         }
 221
 222         return dev;
 223 }
 224
 225 static int
 226 ree_dev_unregister(struct rte_regexdev *dev)
 227 {
 228         cn9k_ree_dbg("Closing regex device %s", dev->device->name);
 229
 230         /* free regex device */
 231         rte_regexdev_unregister(dev);
 232
 233         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 234                 rte_free(dev->data->dev_private);
 235
 236         return 0;
 237 }
 238
 239 static int
 240 ree_dev_fini(struct rte_regexdev *dev)
 241 {
 242         struct cn9k_ree_data *data = dev->data->dev_private;
 243         struct roc_ree_vf *vf = &data->vf;
 244         int i, ret;
 245
 246         ree_func_trace();
 247
 248         for (i = 0; i < data->nb_queue_pairs; i++) {
 249                 ret = ree_queue_pair_release(dev, i);
 250                 if (ret)
 251                         return ret;
 252         }
 253
 254         ret = roc_ree_queues_detach(vf);
 255         if (ret)
 256                 cn9k_err("Could not detach queues");
 257
 258         /* TEMP : should be in lib */
 259         rte_free(data->queue_pairs);
 260         rte_free(data->rules);
 261
 262         roc_ree_dev_fini(vf);
 263
 264         ret = ree_dev_unregister(dev);
 265         if (ret)
 266                 cn9k_err("Could not destroy PMD");
 267
 268         return ret;
 269 }
 270
 271 static inline int
 272 ree_enqueue(struct roc_ree_qp *qp, struct rte_regex_ops *op,
 273                  struct roc_ree_pending_queue *pend_q)
 274 {
 275         union roc_ree_inst inst;
 276         union ree_res *res;
 277         uint32_t offset;
 278
 279         if (unlikely(pend_q->pending_count >= REE_DEFAULT_CMD_QLEN)) {
 280                 cn9k_err("Pending count %" PRIu64 " is greater than Q size %d",
 281                 pend_q->pending_count, REE_DEFAULT_CMD_QLEN);
 282                 return -EAGAIN;
 283         }
 284         if (unlikely(op->mbuf->data_len > REE_MAX_PAYLOAD_SIZE ||
 285                         op->mbuf->data_len == 0)) {
 286                 cn9k_err("Packet length %d is greater than MAX payload %d",
 287                                 op->mbuf->data_len, REE_MAX_PAYLOAD_SIZE);
 288                 return -EAGAIN;
 289         }
 290
 291         /* W 0 */
 292         inst.cn98xx.ooj = 1;
 293         inst.cn98xx.dg = 0;
 294         inst.cn98xx.doneint = 0;
 295         /* W 1 */
 296         inst.cn98xx.inp_ptr_addr = rte_pktmbuf_mtod(op->mbuf, uint64_t);
 297         /* W 2 */
 298         inst.cn98xx.inp_ptr_ctl = op->mbuf->data_len & 0x7FFF;
 299         inst.cn98xx.inp_ptr_ctl = inst.cn98xx.inp_ptr_ctl << 32;
 300
 301         /* W 3 */
 302         inst.cn98xx.res_ptr_addr = (uint64_t)op;
 303         /* W 4 */
 304         inst.cn98xx.wq_ptr = 0;
 305         /* W 5 */
 306         inst.cn98xx.ggrp = 0;
 307         inst.cn98xx.tt = 0;
 308         inst.cn98xx.tag = 0;
 309         /* W 6 */
 310         inst.cn98xx.ree_job_length = op->mbuf->data_len & 0x7FFF;
 311         if (op->req_flags & RTE_REGEX_OPS_REQ_STOP_ON_MATCH_F)
 312                 inst.cn98xx.ree_job_ctrl = (0x2 << 8);
 313         else if (op->req_flags & RTE_REGEX_OPS_REQ_MATCH_HIGH_PRIORITY_F)
 314                 inst.cn98xx.ree_job_ctrl = (0x1 << 8);
 315         else
 316                 inst.cn98xx.ree_job_ctrl = 0;
 317         inst.cn98xx.ree_job_id = qp->roc_regexdev_jobid;
 318         /* W 7 */
 319         inst.cn98xx.ree_job_subset_id_0 = op->group_id0;
 320         if (op->req_flags & RTE_REGEX_OPS_REQ_GROUP_ID1_VALID_F)
 321                 inst.cn98xx.ree_job_subset_id_1 = op->group_id1;
 322         else
 323                 inst.cn98xx.ree_job_subset_id_1 = op->group_id0;
 324         if (op->req_flags & RTE_REGEX_OPS_REQ_GROUP_ID2_VALID_F)
 325                 inst.cn98xx.ree_job_subset_id_2 = op->group_id2;
 326         else
 327                 inst.cn98xx.ree_job_subset_id_2 = op->group_id0;
 328         if (op->req_flags & RTE_REGEX_OPS_REQ_GROUP_ID3_VALID_F)
 329                 inst.cn98xx.ree_job_subset_id_3 = op->group_id3;
 330         else
 331                 inst.cn98xx.ree_job_subset_id_3 = op->group_id0;
 332
 333         /* Copy REE command to Q */
 334         offset = qp->write_offset * sizeof(inst);
 335         memcpy((void *)(qp->iq_dma_addr + offset), &inst, sizeof(inst));
 336
 337         pend_q->rid_queue[pend_q->enq_tail].rid = (uintptr_t)op;
 338         pend_q->rid_queue[pend_q->enq_tail].user_id = op->user_id;
 339
 340         /* Mark result as not done */
 341         res = (union ree_res *)(op);
 342         res->s.done = 0;
 343         res->s.ree_err = 0;
 344
 345         /* We will use soft queue length here to limit requests */
 346         REE_MOD_INC(pend_q->enq_tail, REE_DEFAULT_CMD_QLEN);
 347         pend_q->pending_count += 1;
 348         REE_MOD_INC(qp->roc_regexdev_jobid, 0xFFFFFF);
 349         REE_MOD_INC(qp->write_offset, REE_IQ_LEN);
 350
 351         return 0;
 352 }
 353
 354 static uint16_t
 355 cn9k_ree_enqueue_burst(struct rte_regexdev *dev, uint16_t qp_id,
 356                        struct rte_regex_ops **ops, uint16_t nb_ops)
 357 {
 358         struct cn9k_ree_data *data = dev->data->dev_private;
 359         struct roc_ree_qp *qp = data->queue_pairs[qp_id];
 360         struct roc_ree_pending_queue *pend_q;
 361         uint16_t nb_allowed, count = 0;
 362         struct rte_regex_ops *op;
 363         int ret;
 364
 365         pend_q = &qp->pend_q;
 366
 367         nb_allowed = REE_DEFAULT_CMD_QLEN - pend_q->pending_count;
 368         if (nb_ops > nb_allowed)
 369                 nb_ops = nb_allowed;
 370
 371         for (count = 0; count < nb_ops; count++) {
 372                 op = ops[count];
 373                 ret = ree_enqueue(qp, op, pend_q);
 374
 375                 if (unlikely(ret))
 376                         break;
 377         }
 378
 379         /*
 380          * Make sure all instructions are written before DOORBELL is activated
 381          */
 382         rte_io_wmb();
 383
 384         /* Update Doorbell */
 385         plt_write64(count, qp->base + REE_LF_DOORBELL);
 386
 387         return count;
 388 }
 389
 390 static inline void
 391 ree_dequeue_post_process(struct rte_regex_ops *ops)
 392 {
 393         uint8_t ree_res_mcnt, ree_res_dmcnt;
 394         int off = REE_MATCH_OFFSET;
 395         struct ree_res_s_98 *res;
 396         uint16_t ree_res_status;
 397         uint64_t match;
 398
 399         res = (struct ree_res_s_98 *)ops;
 400         /* store res values on stack since ops and res
 401          * are using the same memory
 402          */
 403         ree_res_status = res->ree_res_status;
 404         ree_res_mcnt = res->ree_res_mcnt;
 405         ree_res_dmcnt = res->ree_res_dmcnt;
 406         ops->rsp_flags = 0;
 407         ops->nb_actual_matches = ree_res_dmcnt;
 408         ops->nb_matches = ree_res_mcnt;
 409         if (unlikely(res->ree_err)) {
 410                 ops->nb_actual_matches = 0;
 411                 ops->nb_matches = 0;
 412         }
 413
 414         if (unlikely(ree_res_status != REE_TYPE_RESULT_DESC)) {
 415                 if (ree_res_status & REE_STATUS_PMI_SOJ_BIT)
 416                         ops->rsp_flags |= RTE_REGEX_OPS_RSP_PMI_SOJ_F;
 417                 if (ree_res_status & REE_STATUS_PMI_EOJ_BIT)
 418                         ops->rsp_flags |= RTE_REGEX_OPS_RSP_PMI_EOJ_F;
 419                 if (ree_res_status & REE_STATUS_ML_CNT_DET_BIT)
 420                         ops->rsp_flags |= RTE_REGEX_OPS_RSP_MAX_SCAN_TIMEOUT_F;
 421                 if (ree_res_status & REE_STATUS_MM_CNT_DET_BIT)
 422                         ops->rsp_flags |= RTE_REGEX_OPS_RSP_MAX_MATCH_F;
 423                 if (ree_res_status & REE_STATUS_MP_CNT_DET_BIT)
 424                         ops->rsp_flags |= RTE_REGEX_OPS_RSP_MAX_PREFIX_F;
 425         }
 426         if (ops->nb_matches > 0) {
 427                 /* Move the matches to the correct offset */
 428                 off = ((ops->nb_matches < REE_NUM_MATCHES_ALIGN) ?
 429                         ops->nb_matches : REE_NUM_MATCHES_ALIGN);
 430                 match = (uint64_t)ops + REE_MATCH_OFFSET;
 431                 match += (ops->nb_matches - off) *
 432                         sizeof(union ree_match);
 433                 memcpy((void *)ops->matches, (void *)match,
 434                         off * sizeof(union ree_match));
 435         }
 436 }
 437
 438 static uint16_t
 439 cn9k_ree_dequeue_burst(struct rte_regexdev *dev, uint16_t qp_id,
 440                        struct rte_regex_ops **ops, uint16_t nb_ops)
 441 {
 442         struct cn9k_ree_data *data = dev->data->dev_private;
 443         struct roc_ree_qp *qp = data->queue_pairs[qp_id];
 444         struct roc_ree_pending_queue *pend_q;
 445         int i, nb_pending, nb_completed = 0;
 446         volatile struct ree_res_s_98 *res;
 447         struct roc_ree_rid *rid;
 448
 449         pend_q = &qp->pend_q;
 450
 451         nb_pending = pend_q->pending_count;
 452
 453         if (nb_ops > nb_pending)
 454                 nb_ops = nb_pending;
 455
 456         for (i = 0; i < nb_ops; i++) {
 457                 rid = &pend_q->rid_queue[pend_q->deq_head];
 458                 res = (volatile struct ree_res_s_98 *)(rid->rid);
 459
 460                 /* Check response header done bit if completed */
 461                 if (unlikely(!res->done))
 462                         break;
 463
 464                 ops[i] = (struct rte_regex_ops *)(rid->rid);
 465                 ops[i]->user_id = rid->user_id;
 466
 467                 REE_MOD_INC(pend_q->deq_head, REE_DEFAULT_CMD_QLEN);
 468                 pend_q->pending_count -= 1;
 469         }
 470
 471         nb_completed = i;
 472
 473         for (i = 0; i < nb_completed; i++)
 474                 ree_dequeue_post_process(ops[i]);
 475
 476         return nb_completed;
 477 }
 478
 479 static int
 480 cn9k_ree_dev_info_get(struct rte_regexdev *dev, struct rte_regexdev_info *info)
 481 {
 482         struct cn9k_ree_data *data = dev->data->dev_private;
 483         struct roc_ree_vf *vf = &data->vf;
 484
 485         ree_func_trace();
 486
 487         if (info == NULL)
 488                 return -EINVAL;
 489
 490         info->driver_name = dev->device->driver->name;
 491         info->dev = dev->device;
 492
 493         info->max_queue_pairs = vf->max_queues;
 494         info->max_matches = vf->max_matches;
 495         info->max_payload_size = REE_MAX_PAYLOAD_SIZE;
 496         info->max_rules_per_group = data->max_rules_per_group;
 497         info->max_groups = data->max_groups;
 498         info->regexdev_capa = data->regexdev_capa;
 499         info->rule_flags = data->rule_flags;
 500
 501         return 0;
 502 }
 503
 504 static int
 505 cn9k_ree_dev_config(struct rte_regexdev *dev,
 506                     const struct rte_regexdev_config *cfg)
 507 {
 508         struct cn9k_ree_data *data = dev->data->dev_private;
 509         struct roc_ree_vf *vf = &data->vf;
 510         const struct ree_rule_db *rule_db;
 511         uint32_t rule_db_len;
 512         int ret;
 513
 514         ree_func_trace();
 515
 516         if (cfg->nb_queue_pairs > vf->max_queues) {
 517                 cn9k_err("Invalid number of queue pairs requested");
 518                 return -EINVAL;
 519         }
 520
 521         if (cfg->nb_max_matches != vf->max_matches) {
 522                 cn9k_err("Invalid number of max matches requested");
 523                 return -EINVAL;
 524         }
 525
 526         if (cfg->dev_cfg_flags != 0) {
 527                 cn9k_err("Invalid device configuration flags requested");
 528                 return -EINVAL;
 529         }
 530
 531         /* Unregister error interrupts */
 532         if (vf->err_intr_registered)
 533                 roc_ree_err_intr_unregister(vf);
 534
 535         /* Detach queues */
 536         if (vf->nb_queues) {
 537                 ret = roc_ree_queues_detach(vf);
 538                 if (ret) {
 539                         cn9k_err("Could not detach REE queues");
 540                         return ret;
 541                 }
 542         }
 543
 544         /* TEMP : should be in lib */
 545         if (data->queue_pairs == NULL) { /* first time configuration */
 546                 data->queue_pairs = rte_zmalloc("regexdev->queue_pairs",
 547                                 sizeof(data->queue_pairs[0]) *
 548                                 cfg->nb_queue_pairs, RTE_CACHE_LINE_SIZE);
 549
 550                 if (data->queue_pairs == NULL) {
 551                         data->nb_queue_pairs = 0;
 552                         cn9k_err("Failed to get memory for qp meta data, nb_queues %u",
 553                                         cfg->nb_queue_pairs);
 554                         return -ENOMEM;
 555                 }
 556         } else { /* re-configure */
 557                 uint16_t old_nb_queues = data->nb_queue_pairs;
 558                 void **qp;
 559                 unsigned int i;
 560
 561                 qp = data->queue_pairs;
 562
 563                 for (i = cfg->nb_queue_pairs; i < old_nb_queues; i++) {
 564                         ret = ree_queue_pair_release(dev, i);
 565                         if (ret < 0)
 566                                 return ret;
 567                 }
 568
 569                 qp = rte_realloc(qp, sizeof(qp[0]) * cfg->nb_queue_pairs,
 570                                 RTE_CACHE_LINE_SIZE);
 571                 if (qp == NULL) {
 572                         cn9k_err("Failed to realloc qp meta data, nb_queues %u",
 573                                         cfg->nb_queue_pairs);
 574                         return -ENOMEM;
 575                 }
 576
 577                 if (cfg->nb_queue_pairs > old_nb_queues) {
 578                         uint16_t new_qs = cfg->nb_queue_pairs - old_nb_queues;
 579                         memset(qp + old_nb_queues, 0, sizeof(qp[0]) * new_qs);
 580                 }
 581
 582                 data->queue_pairs = qp;
 583         }
 584         data->nb_queue_pairs = cfg->nb_queue_pairs;
 585
 586         /* Attach queues */
 587         cn9k_ree_dbg("Attach %d queues", cfg->nb_queue_pairs);
 588         ret = roc_ree_queues_attach(vf, cfg->nb_queue_pairs);
 589         if (ret) {
 590                 cn9k_err("Could not attach queues");
 591                 return -ENODEV;
 592         }
 593
 594         ret = roc_ree_msix_offsets_get(vf);
 595         if (ret) {
 596                 cn9k_err("Could not get MSI-X offsets");
 597                 goto queues_detach;
 598         }
 599
 600         if (cfg->rule_db && cfg->rule_db_len) {
 601                 cn9k_ree_dbg("rule_db length %d", cfg->rule_db_len);
 602                 rule_db = (const struct ree_rule_db *)cfg->rule_db;
 603                 rule_db_len = rule_db->number_of_entries *
 604                                 sizeof(struct ree_rule_db_entry);
 605                 cn9k_ree_dbg("rule_db number of entries %d",
 606                                 rule_db->number_of_entries);
 607                 if (rule_db_len > cfg->rule_db_len) {
 608                         cn9k_err("Could not program rule db");
 609                         ret = -EINVAL;
 610                         goto queues_detach;
 611                 }
 612                 ret = roc_ree_rule_db_prog(vf, (const char *)rule_db->entries,
 613                                 rule_db_len, NULL, REE_NON_INC_PROG);
 614                 if (ret) {
 615                         cn9k_err("Could not program rule db");
 616                         goto queues_detach;
 617                 }
 618         }
 619
 620         dev->enqueue = cn9k_ree_enqueue_burst;
 621         dev->dequeue = cn9k_ree_dequeue_burst;
 622
 623         rte_mb();
 624         return 0;
 625
 626 queues_detach:
 627         roc_ree_queues_detach(vf);
 628         return ret;
 629 }
 630
 631 static int
 632 cn9k_ree_stop(struct rte_regexdev *dev)
 633 {
 634         RTE_SET_USED(dev);
 635
 636         ree_func_trace();
 637         return 0;
 638 }
 639
 640 static int
 641 cn9k_ree_start(struct rte_regexdev *dev)
 642 {
 643         struct cn9k_ree_data *data = dev->data->dev_private;
 644         struct roc_ree_vf *vf = &data->vf;
 645         uint32_t rule_db_len = 0;
 646         int ret;
 647
 648         ree_func_trace();
 649
 650         ret = roc_ree_rule_db_len_get(vf, &rule_db_len, NULL);
 651         if (ret)
 652                 return ret;
 653         if (rule_db_len == 0) {
 654                 cn9k_err("Rule db not programmed");
 655                 return -EFAULT;
 656         }
 657
 658         return 0;
 659 }
 660
 661 static int
 662 cn9k_ree_close(struct rte_regexdev *dev)
 663 {
 664         return ree_dev_fini(dev);
 665 }
 666
 667 static int
 668 cn9k_ree_queue_pair_setup(struct rte_regexdev *dev, uint16_t qp_id,
 669                 const struct rte_regexdev_qp_conf *qp_conf)
 670 {
 671         struct cn9k_ree_data *data = dev->data->dev_private;
 672         struct roc_ree_qp *qp;
 673
 674         ree_func_trace("Queue=%d", qp_id);
 675
 676         if (data->queue_pairs[qp_id] != NULL)
 677                 ree_queue_pair_release(dev, qp_id);
 678
 679         if (qp_conf->nb_desc > REE_DEFAULT_CMD_QLEN) {
 680                 cn9k_err("Could not setup queue pair for %u descriptors",
 681                                 qp_conf->nb_desc);
 682                 return -EINVAL;
 683         }
 684         if (qp_conf->qp_conf_flags != 0) {
 685                 cn9k_err("Could not setup queue pair with configuration flags 0x%x",
 686                                 qp_conf->qp_conf_flags);
 687                 return -EINVAL;
 688         }
 689
 690         qp = ree_qp_create(dev, qp_id);
 691         if (qp == NULL) {
 692                 cn9k_err("Could not create queue pair %d", qp_id);
 693                 return -ENOMEM;
 694         }
 695         data->queue_pairs[qp_id] = qp;
 696
 697         return 0;
 698 }
 699
 700 static int
 701 cn9k_ree_rule_db_compile_activate(struct rte_regexdev *dev)
 702 {
 703         return cn9k_ree_rule_db_compile_prog(dev);
 704 }
 705
 706 static int
 707 cn9k_ree_rule_db_update(struct rte_regexdev *dev,
 708                 const struct rte_regexdev_rule *rules, uint16_t nb_rules)
 709 {
 710         struct cn9k_ree_data *data = dev->data->dev_private;
 711         struct rte_regexdev_rule *old_ptr;
 712         uint32_t i, sum_nb_rules;
 713
 714         ree_func_trace("nb_rules=%d", nb_rules);
 715
 716         for (i = 0; i < nb_rules; i++) {
 717                 if (rules[i].op == RTE_REGEX_RULE_OP_REMOVE)
 718                         break;
 719                 if (rules[i].group_id >= data->max_groups)
 720                         break;
 721                 if (rules[i].rule_id >= data->max_rules_per_group)
 722                         break;
 723                 /* logical implication
 724                  * p    q    p -> q
 725                  * 0    0      1
 726                  * 0    1      1
 727                  * 1    0      0
 728                  * 1    1      1
 729                  */
 730                 if ((~(rules[i].rule_flags) | data->rule_flags) == 0)
 731                         break;
 732         }
 733         nb_rules = i;
 734
 735         if (data->nb_rules == 0) {
 736
 737                 data->rules = rte_malloc("rte_regexdev_rules",
 738                                 nb_rules*sizeof(struct rte_regexdev_rule), 0);
 739                 if (data->rules == NULL)
 740                         return -ENOMEM;
 741
 742                 memcpy(data->rules, rules,
 743                                 nb_rules*sizeof(struct rte_regexdev_rule));
 744                 data->nb_rules = nb_rules;
 745         } else {
 746
 747                 old_ptr = data->rules;
 748                 sum_nb_rules = data->nb_rules + nb_rules;
 749                 data->rules = rte_realloc(data->rules,
 750                                 sum_nb_rules * sizeof(struct rte_regexdev_rule),
 751                                                         0);
 752                 if (data->rules == NULL) {
 753                         data->rules = old_ptr;
 754                         return -ENOMEM;
 755                 }
 756                 memcpy(&data->rules[data->nb_rules], rules,
 757                                 nb_rules*sizeof(struct rte_regexdev_rule));
 758                 data->nb_rules = sum_nb_rules;
 759         }
 760         return nb_rules;
 761 }
 762
 763 static int
 764 cn9k_ree_rule_db_import(struct rte_regexdev *dev, const char *rule_db,
 765                 uint32_t rule_db_len)
 766 {
 767         struct cn9k_ree_data *data = dev->data->dev_private;
 768         struct roc_ree_vf *vf = &data->vf;
 769         const struct ree_rule_db *ree_rule_db;
 770         uint32_t ree_rule_db_len;
 771         int ret;
 772
 773         ree_func_trace("rule_db_len=%d", rule_db_len);
 774
 775         ree_rule_db = (const struct ree_rule_db *)rule_db;
 776         ree_rule_db_len = ree_rule_db->number_of_entries *
 777                         sizeof(struct ree_rule_db_entry);
 778         if (ree_rule_db_len > rule_db_len) {
 779                 cn9k_err("Could not program rule db");
 780                 return -EINVAL;
 781         }
 782         ret = roc_ree_rule_db_prog(vf, (const char *)ree_rule_db->entries,
 783                         ree_rule_db_len, NULL, REE_NON_INC_PROG);
 784         if (ret) {
 785                 cn9k_err("Could not program rule db");
 786                 return -ENOSPC;
 787         }
 788         return 0;
 789 }
 790
 791 static int
 792 cn9k_ree_rule_db_export(struct rte_regexdev *dev, char *rule_db)
 793 {
 794         struct cn9k_ree_data *data = dev->data->dev_private;
 795         struct roc_ree_vf *vf = &data->vf;
 796         struct ree_rule_db *ree_rule_db;
 797         uint32_t rule_dbi_len;
 798         uint32_t rule_db_len;
 799         int ret;
 800
 801         ree_func_trace();
 802
 803         ret = roc_ree_rule_db_len_get(vf, &rule_db_len, &rule_dbi_len);
 804         if (ret)
 805                 return ret;
 806
 807         if (rule_db == NULL) {
 808                 rule_db_len += sizeof(struct ree_rule_db);
 809                 return rule_db_len;
 810         }
 811
 812         ree_rule_db = (struct ree_rule_db *)rule_db;
 813         ret = roc_ree_rule_db_get(vf, (char *)ree_rule_db->entries,
 814                         rule_db_len, NULL, 0);
 815         if (ret) {
 816                 cn9k_err("Could not export rule db");
 817                 return -EFAULT;
 818         }
 819         ree_rule_db->number_of_entries =
 820                         rule_db_len/sizeof(struct ree_rule_db_entry);
 821         ree_rule_db->revision = REE_RULE_DB_REVISION;
 822         ree_rule_db->version = REE_RULE_DB_VERSION;
 823
 824         return 0;
 825 }
 826
 827 static struct rte_regexdev_ops cn9k_ree_ops = {
 828         .dev_info_get = cn9k_ree_dev_info_get,
 829         .dev_configure = cn9k_ree_dev_config,
 830         .dev_qp_setup = cn9k_ree_queue_pair_setup,
 831         .dev_start = cn9k_ree_start,
 832         .dev_stop = cn9k_ree_stop,
 833         .dev_close = cn9k_ree_close,
 834         .dev_attr_get = NULL,
 835         .dev_attr_set = NULL,
 836         .dev_rule_db_update = cn9k_ree_rule_db_update,
 837         .dev_rule_db_compile_activate =
 838                         cn9k_ree_rule_db_compile_activate,
 839         .dev_db_import = cn9k_ree_rule_db_import,
 840         .dev_db_export = cn9k_ree_rule_db_export,
 841         .dev_xstats_names_get = NULL,
 842         .dev_xstats_get = NULL,
 843         .dev_xstats_by_name_get = NULL,
 844         .dev_xstats_reset = NULL,
 845         .dev_selftest = NULL,
 846         .dev_dump = NULL,
 847 };
 848
 849 static int
 850 cn9k_ree_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 851                    struct rte_pci_device *pci_dev)
 852 {
 853         char name[RTE_REGEXDEV_NAME_MAX_LEN];
 854         struct cn9k_ree_data *data;
 855         struct rte_regexdev *dev;
 856         struct roc_ree_vf *vf;
 857         int ret;
 858
 859         ret = roc_plt_init();
 860         if (ret < 0) {
 861                 plt_err("Failed to initialize platform model");
 862                 return ret;
 863         }
 864
 865         rte_pci_device_name(&pci_dev->addr, name, sizeof(name));
 866
 867         dev = ree_dev_register(name);
 868         if (dev == NULL) {
 869                 ret = -ENODEV;
 870                 goto exit;
 871         }
 872
 873         dev->dev_ops = &cn9k_ree_ops;
 874         dev->device = &pci_dev->device;
 875
 876         /* Get private data space allocated */
 877         data = dev->data->dev_private;
 878         vf = &data->vf;
 879         vf->pci_dev = pci_dev;
 880         ret = roc_ree_dev_init(vf);
 881         if (ret) {
 882                 plt_err("Failed to initialize roc cpt rc=%d", ret);
 883                 goto dev_unregister;
 884         }
 885
 886         data->rule_flags = RTE_REGEX_PCRE_RULE_ALLOW_EMPTY_F |
 887                         RTE_REGEX_PCRE_RULE_ANCHORED_F;
 888         data->regexdev_capa = 0;
 889         data->max_groups = REE_MAX_GROUPS;
 890         data->max_rules_per_group = REE_MAX_RULES_PER_GROUP;
 891         data->nb_rules = 0;
 892
 893         dev->state = RTE_REGEXDEV_READY;
 894         return 0;
 895
 896 dev_unregister:
 897         ree_dev_unregister(dev);
 898 exit:
 899         cn9k_err("Could not create device (vendor_id: 0x%x device_id: 0x%x)",
 900                     pci_dev->id.vendor_id, pci_dev->id.device_id);
 901         return ret;
 902 }
 903
 904 static int
 905 cn9k_ree_pci_remove(struct rte_pci_device *pci_dev)
 906 {
 907         char name[RTE_REGEXDEV_NAME_MAX_LEN];
 908         struct rte_regexdev *dev = NULL;
 909
 910         if (pci_dev == NULL)
 911                 return -EINVAL;
 912
 913         rte_pci_device_name(&pci_dev->addr, name, sizeof(name));
 914
 915         dev = rte_regexdev_get_device_by_name(name);
 916
 917         if (dev == NULL)
 918                 return -ENODEV;
 919
 920         return ree_dev_fini(dev);
 921 }
 922
 923 static struct rte_pci_id pci_id_ree_table[] = {
 924         {
 925                 RTE_PCI_DEVICE(PCI_VENDOR_ID_CAVIUM,
 926                                 PCI_DEVID_CNXK_RVU_REE_PF)
 927         },
 928         {
 929                 .vendor_id = 0,
 930         }
 931 };
 932
 933 static struct rte_pci_driver cn9k_regexdev_pmd = {
 934         .id_table = pci_id_ree_table,
 935         .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
 936         .probe = cn9k_ree_pci_probe,
 937         .remove = cn9k_ree_pci_remove,
 938 };
 939
 940
 941 RTE_PMD_REGISTER_PCI(REGEXDEV_NAME_CN9K_PMD, cn9k_regexdev_pmd);
 942 RTE_PMD_REGISTER_PCI_TABLE(REGEXDEV_NAME_CN9K_PMD, pci_id_ree_table);