drivers/net/af_packet/rte_eth_af_packet.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2014 John W. Linville <linville@tuxdriver.com>
   3  * Originally based upon librte_pmd_pcap code:
   4  * Copyright(c) 2010-2015 Intel Corporation.
   5  * Copyright(c) 2014 6WIND S.A.
   6  * All rights reserved.
   7  */
   8
   9 #include <rte_string_fns.h>
  10 #include <rte_mbuf.h>
  11 #include <rte_ethdev_driver.h>
  12 #include <rte_ethdev_vdev.h>
  13 #include <rte_malloc.h>
  14 #include <rte_kvargs.h>
  15 #include <rte_bus_vdev.h>
  16
  17 #include <errno.h>
  18 #include <linux/if_ether.h>
  19 #include <linux/if_packet.h>
  20 #include <arpa/inet.h>
  21 #include <net/if.h>
  22 #include <net/if_arp.h>
  23 #include <sys/types.h>
  24 #include <sys/socket.h>
  25 #include <sys/ioctl.h>
  26 #include <string.h>
  27 #include <sys/mman.h>
  28 #include <unistd.h>
  29 #include <poll.h>
  30
  31 #define ETH_AF_PACKET_IFACE_ARG         "iface"
  32 #define ETH_AF_PACKET_NUM_Q_ARG         "qpairs"
  33 #define ETH_AF_PACKET_BLOCKSIZE_ARG     "blocksz"
  34 #define ETH_AF_PACKET_FRAMESIZE_ARG     "framesz"
  35 #define ETH_AF_PACKET_FRAMECOUNT_ARG    "framecnt"
  36 #define ETH_AF_PACKET_QDISC_BYPASS_ARG  "qdisc_bypass"
  37
  38 #define DFLT_FRAME_SIZE         (1 << 11)
  39 #define DFLT_FRAME_COUNT        (1 << 9)
  40
  41 #define RTE_PMD_AF_PACKET_MAX_RINGS 16
  42
  43 struct pkt_rx_queue {
  44         int sockfd;
  45
  46         struct iovec *rd;
  47         uint8_t *map;
  48         unsigned int framecount;
  49         unsigned int framenum;
  50
  51         struct rte_mempool *mb_pool;
  52         uint16_t in_port;
  53
  54         volatile unsigned long rx_pkts;
  55         volatile unsigned long rx_bytes;
  56 };
  57
  58 struct pkt_tx_queue {
  59         int sockfd;
  60         unsigned int frame_data_size;
  61
  62         struct iovec *rd;
  63         uint8_t *map;
  64         unsigned int framecount;
  65         unsigned int framenum;
  66
  67         volatile unsigned long tx_pkts;
  68         volatile unsigned long err_pkts;
  69         volatile unsigned long tx_bytes;
  70 };
  71
  72 struct pmd_internals {
  73         unsigned nb_queues;
  74
  75         int if_index;
  76         char *if_name;
  77         struct rte_ether_addr eth_addr;
  78
  79         struct tpacket_req req;
  80
  81         struct pkt_rx_queue rx_queue[RTE_PMD_AF_PACKET_MAX_RINGS];
  82         struct pkt_tx_queue tx_queue[RTE_PMD_AF_PACKET_MAX_RINGS];
  83 };
  84
  85 static const char *valid_arguments[] = {
  86         ETH_AF_PACKET_IFACE_ARG,
  87         ETH_AF_PACKET_NUM_Q_ARG,
  88         ETH_AF_PACKET_BLOCKSIZE_ARG,
  89         ETH_AF_PACKET_FRAMESIZE_ARG,
  90         ETH_AF_PACKET_FRAMECOUNT_ARG,
  91         ETH_AF_PACKET_QDISC_BYPASS_ARG,
  92         NULL
  93 };
  94
  95 static struct rte_eth_link pmd_link = {
  96         .link_speed = ETH_SPEED_NUM_10G,
  97         .link_duplex = ETH_LINK_FULL_DUPLEX,
  98         .link_status = ETH_LINK_DOWN,
  99         .link_autoneg = ETH_LINK_FIXED,
 100 };
 101
 102 static int af_packet_logtype;
 103
 104 #define PMD_LOG(level, fmt, args...) \
 105         rte_log(RTE_LOG_ ## level, af_packet_logtype, \
 106                 "%s(): " fmt "\n", __func__, ##args)
 107
 108 #define PMD_LOG_ERRNO(level, fmt, args...) \
 109         rte_log(RTE_LOG_ ## level, af_packet_logtype, \
 110                 "%s(): " fmt ":%s\n", __func__, ##args, strerror(errno))
 111
 112 static uint16_t
 113 eth_af_packet_rx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 114 {
 115         unsigned i;
 116         struct tpacket2_hdr *ppd;
 117         struct rte_mbuf *mbuf;
 118         uint8_t *pbuf;
 119         struct pkt_rx_queue *pkt_q = queue;
 120         uint16_t num_rx = 0;
 121         unsigned long num_rx_bytes = 0;
 122         unsigned int framecount, framenum;
 123
 124         if (unlikely(nb_pkts == 0))
 125                 return 0;
 126
 127         /*
 128          * Reads the given number of packets from the AF_PACKET socket one by
 129          * one and copies the packet data into a newly allocated mbuf.
 130          */
 131         framecount = pkt_q->framecount;
 132         framenum = pkt_q->framenum;
 133         for (i = 0; i < nb_pkts; i++) {
 134                 /* point at the next incoming frame */
 135                 ppd = (struct tpacket2_hdr *) pkt_q->rd[framenum].iov_base;
 136                 if ((ppd->tp_status & TP_STATUS_USER) == 0)
 137                         break;
 138
 139                 /* allocate the next mbuf */
 140                 mbuf = rte_pktmbuf_alloc(pkt_q->mb_pool);
 141                 if (unlikely(mbuf == NULL))
 142                         break;
 143
 144                 /* packet will fit in the mbuf, go ahead and receive it */
 145                 rte_pktmbuf_pkt_len(mbuf) = rte_pktmbuf_data_len(mbuf) = ppd->tp_snaplen;
 146                 pbuf = (uint8_t *) ppd + ppd->tp_mac;
 147                 memcpy(rte_pktmbuf_mtod(mbuf, void *), pbuf, rte_pktmbuf_data_len(mbuf));
 148
 149                 /* check for vlan info */
 150                 if (ppd->tp_status & TP_STATUS_VLAN_VALID) {
 151                         mbuf->vlan_tci = ppd->tp_vlan_tci;
 152                         mbuf->ol_flags |= (PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED);
 153                 }
 154
 155                 /* release incoming frame and advance ring buffer */
 156                 ppd->tp_status = TP_STATUS_KERNEL;
 157                 if (++framenum >= framecount)
 158                         framenum = 0;
 159                 mbuf->port = pkt_q->in_port;
 160
 161                 /* account for the receive frame */
 162                 bufs[i] = mbuf;
 163                 num_rx++;
 164                 num_rx_bytes += mbuf->pkt_len;
 165         }
 166         pkt_q->framenum = framenum;
 167         pkt_q->rx_pkts += num_rx;
 168         pkt_q->rx_bytes += num_rx_bytes;
 169         return num_rx;
 170 }
 171
 172 /*
 173  * Callback to handle sending packets through a real NIC.
 174  */
 175 static uint16_t
 176 eth_af_packet_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 177 {
 178         struct tpacket2_hdr *ppd;
 179         struct rte_mbuf *mbuf;
 180         uint8_t *pbuf;
 181         unsigned int framecount, framenum;
 182         struct pollfd pfd;
 183         struct pkt_tx_queue *pkt_q = queue;
 184         uint16_t num_tx = 0;
 185         unsigned long num_tx_bytes = 0;
 186         int i;
 187
 188         if (unlikely(nb_pkts == 0))
 189                 return 0;
 190
 191         memset(&pfd, 0, sizeof(pfd));
 192         pfd.fd = pkt_q->sockfd;
 193         pfd.events = POLLOUT;
 194         pfd.revents = 0;
 195
 196         framecount = pkt_q->framecount;
 197         framenum = pkt_q->framenum;
 198         ppd = (struct tpacket2_hdr *) pkt_q->rd[framenum].iov_base;
 199         for (i = 0; i < nb_pkts; i++) {
 200                 mbuf = *bufs++;
 201
 202                 /* drop oversized packets */
 203                 if (mbuf->pkt_len > pkt_q->frame_data_size) {
 204                         rte_pktmbuf_free(mbuf);
 205                         continue;
 206                 }
 207
 208                 /* insert vlan info if necessary */
 209                 if (mbuf->ol_flags & PKT_TX_VLAN_PKT) {
 210                         if (rte_vlan_insert(&mbuf)) {
 211                                 rte_pktmbuf_free(mbuf);
 212                                 continue;
 213                         }
 214                 }
 215
 216                 /* point at the next incoming frame */
 217                 if ((ppd->tp_status != TP_STATUS_AVAILABLE) &&
 218                     (poll(&pfd, 1, -1) < 0))
 219                         break;
 220
 221                 /* copy the tx frame data */
 222                 pbuf = (uint8_t *) ppd + TPACKET2_HDRLEN -
 223                         sizeof(struct sockaddr_ll);
 224
 225                 struct rte_mbuf *tmp_mbuf = mbuf;
 226                 while (tmp_mbuf) {
 227                         uint16_t data_len = rte_pktmbuf_data_len(tmp_mbuf);
 228                         memcpy(pbuf, rte_pktmbuf_mtod(tmp_mbuf, void*), data_len);
 229                         pbuf += data_len;
 230                         tmp_mbuf = tmp_mbuf->next;
 231                 }
 232
 233                 ppd->tp_len = mbuf->pkt_len;
 234                 ppd->tp_snaplen = mbuf->pkt_len;
 235
 236                 /* release incoming frame and advance ring buffer */
 237                 ppd->tp_status = TP_STATUS_SEND_REQUEST;
 238                 if (++framenum >= framecount)
 239                         framenum = 0;
 240                 ppd = (struct tpacket2_hdr *) pkt_q->rd[framenum].iov_base;
 241
 242                 num_tx++;
 243                 num_tx_bytes += mbuf->pkt_len;
 244                 rte_pktmbuf_free(mbuf);
 245         }
 246
 247         /* kick-off transmits */
 248         if (sendto(pkt_q->sockfd, NULL, 0, MSG_DONTWAIT, NULL, 0) == -1 &&
 249                         errno != ENOBUFS && errno != EAGAIN) {
 250                 /*
 251                  * In case of a ENOBUFS/EAGAIN error all of the enqueued
 252                  * packets will be considered successful even though only some
 253                  * are sent.
 254                  */
 255
 256                 num_tx = 0;
 257                 num_tx_bytes = 0;
 258         }
 259
 260         pkt_q->framenum = framenum;
 261         pkt_q->tx_pkts += num_tx;
 262         pkt_q->err_pkts += i - num_tx;
 263         pkt_q->tx_bytes += num_tx_bytes;
 264         return i;
 265 }
 266
 267 static int
 268 eth_dev_start(struct rte_eth_dev *dev)
 269 {
 270         dev->data->dev_link.link_status = ETH_LINK_UP;
 271         return 0;
 272 }
 273
 274 /*
 275  * This function gets called when the current port gets stopped.
 276  */
 277 static void
 278 eth_dev_stop(struct rte_eth_dev *dev)
 279 {
 280         unsigned i;
 281         int sockfd;
 282         struct pmd_internals *internals = dev->data->dev_private;
 283
 284         for (i = 0; i < internals->nb_queues; i++) {
 285                 sockfd = internals->rx_queue[i].sockfd;
 286                 if (sockfd != -1)
 287                         close(sockfd);
 288
 289                 /* Prevent use after free in case tx fd == rx fd */
 290                 if (sockfd != internals->tx_queue[i].sockfd) {
 291                         sockfd = internals->tx_queue[i].sockfd;
 292                         if (sockfd != -1)
 293                                 close(sockfd);
 294                 }
 295
 296                 internals->rx_queue[i].sockfd = -1;
 297                 internals->tx_queue[i].sockfd = -1;
 298         }
 299
 300         dev->data->dev_link.link_status = ETH_LINK_DOWN;
 301 }
 302
 303 static int
 304 eth_dev_configure(struct rte_eth_dev *dev __rte_unused)
 305 {
 306         return 0;
 307 }
 308
 309 static int
 310 eth_dev_info(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
 311 {
 312         struct pmd_internals *internals = dev->data->dev_private;
 313
 314         dev_info->if_index = internals->if_index;
 315         dev_info->max_mac_addrs = 1;
 316         dev_info->max_rx_pktlen = (uint32_t)ETH_FRAME_LEN;
 317         dev_info->max_rx_queues = (uint16_t)internals->nb_queues;
 318         dev_info->max_tx_queues = (uint16_t)internals->nb_queues;
 319         dev_info->min_rx_bufsize = 0;
 320         dev_info->tx_offload_capa = DEV_TX_OFFLOAD_MULTI_SEGS |
 321                 DEV_TX_OFFLOAD_VLAN_INSERT;
 322
 323         return 0;
 324 }
 325
 326 static int
 327 eth_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *igb_stats)
 328 {
 329         unsigned i, imax;
 330         unsigned long rx_total = 0, tx_total = 0, tx_err_total = 0;
 331         unsigned long rx_bytes_total = 0, tx_bytes_total = 0;
 332         const struct pmd_internals *internal = dev->data->dev_private;
 333
 334         imax = (internal->nb_queues < RTE_ETHDEV_QUEUE_STAT_CNTRS ?
 335                 internal->nb_queues : RTE_ETHDEV_QUEUE_STAT_CNTRS);
 336         for (i = 0; i < imax; i++) {
 337                 igb_stats->q_ipackets[i] = internal->rx_queue[i].rx_pkts;
 338                 igb_stats->q_ibytes[i] = internal->rx_queue[i].rx_bytes;
 339                 rx_total += igb_stats->q_ipackets[i];
 340                 rx_bytes_total += igb_stats->q_ibytes[i];
 341         }
 342
 343         imax = (internal->nb_queues < RTE_ETHDEV_QUEUE_STAT_CNTRS ?
 344                 internal->nb_queues : RTE_ETHDEV_QUEUE_STAT_CNTRS);
 345         for (i = 0; i < imax; i++) {
 346                 igb_stats->q_opackets[i] = internal->tx_queue[i].tx_pkts;
 347                 igb_stats->q_obytes[i] = internal->tx_queue[i].tx_bytes;
 348                 tx_total += igb_stats->q_opackets[i];
 349                 tx_err_total += internal->tx_queue[i].err_pkts;
 350                 tx_bytes_total += igb_stats->q_obytes[i];
 351         }
 352
 353         igb_stats->ipackets = rx_total;
 354         igb_stats->ibytes = rx_bytes_total;
 355         igb_stats->opackets = tx_total;
 356         igb_stats->oerrors = tx_err_total;
 357         igb_stats->obytes = tx_bytes_total;
 358         return 0;
 359 }
 360
 361 static int
 362 eth_stats_reset(struct rte_eth_dev *dev)
 363 {
 364         unsigned i;
 365         struct pmd_internals *internal = dev->data->dev_private;
 366
 367         for (i = 0; i < internal->nb_queues; i++) {
 368                 internal->rx_queue[i].rx_pkts = 0;
 369                 internal->rx_queue[i].rx_bytes = 0;
 370         }
 371
 372         for (i = 0; i < internal->nb_queues; i++) {
 373                 internal->tx_queue[i].tx_pkts = 0;
 374                 internal->tx_queue[i].err_pkts = 0;
 375                 internal->tx_queue[i].tx_bytes = 0;
 376         }
 377
 378         return 0;
 379 }
 380
 381 static void
 382 eth_dev_close(struct rte_eth_dev *dev __rte_unused)
 383 {
 384 }
 385
 386 static void
 387 eth_queue_release(void *q __rte_unused)
 388 {
 389 }
 390
 391 static int
 392 eth_link_update(struct rte_eth_dev *dev __rte_unused,
 393                 int wait_to_complete __rte_unused)
 394 {
 395         return 0;
 396 }
 397
 398 static int
 399 eth_rx_queue_setup(struct rte_eth_dev *dev,
 400                    uint16_t rx_queue_id,
 401                    uint16_t nb_rx_desc __rte_unused,
 402                    unsigned int socket_id __rte_unused,
 403                    const struct rte_eth_rxconf *rx_conf __rte_unused,
 404                    struct rte_mempool *mb_pool)
 405 {
 406         struct pmd_internals *internals = dev->data->dev_private;
 407         struct pkt_rx_queue *pkt_q = &internals->rx_queue[rx_queue_id];
 408         unsigned int buf_size, data_size;
 409
 410         pkt_q->mb_pool = mb_pool;
 411
 412         /* Now get the space available for data in the mbuf */
 413         buf_size = rte_pktmbuf_data_room_size(pkt_q->mb_pool) -
 414                 RTE_PKTMBUF_HEADROOM;
 415         data_size = internals->req.tp_frame_size;
 416         data_size -= TPACKET2_HDRLEN - sizeof(struct sockaddr_ll);
 417
 418         if (data_size > buf_size) {
 419                 PMD_LOG(ERR,
 420                         "%s: %d bytes will not fit in mbuf (%d bytes)",
 421                         dev->device->name, data_size, buf_size);
 422                 return -ENOMEM;
 423         }
 424
 425         dev->data->rx_queues[rx_queue_id] = pkt_q;
 426         pkt_q->in_port = dev->data->port_id;
 427
 428         return 0;
 429 }
 430
 431 static int
 432 eth_tx_queue_setup(struct rte_eth_dev *dev,
 433                    uint16_t tx_queue_id,
 434                    uint16_t nb_tx_desc __rte_unused,
 435                    unsigned int socket_id __rte_unused,
 436                    const struct rte_eth_txconf *tx_conf __rte_unused)
 437 {
 438
 439         struct pmd_internals *internals = dev->data->dev_private;
 440
 441         dev->data->tx_queues[tx_queue_id] = &internals->tx_queue[tx_queue_id];
 442         return 0;
 443 }
 444
 445 static int
 446 eth_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
 447 {
 448         struct pmd_internals *internals = dev->data->dev_private;
 449         struct ifreq ifr = { .ifr_mtu = mtu };
 450         int ret;
 451         int s;
 452         unsigned int data_size = internals->req.tp_frame_size -
 453                                  TPACKET2_HDRLEN;
 454
 455         if (mtu > data_size)
 456                 return -EINVAL;
 457
 458         s = socket(PF_INET, SOCK_DGRAM, 0);
 459         if (s < 0)
 460                 return -EINVAL;
 461
 462         strlcpy(ifr.ifr_name, internals->if_name, IFNAMSIZ);
 463         ret = ioctl(s, SIOCSIFMTU, &ifr);
 464         close(s);
 465
 466         if (ret < 0)
 467                 return -EINVAL;
 468
 469         return 0;
 470 }
 471
 472 static int
 473 eth_dev_macaddr_set(struct rte_eth_dev *dev, struct rte_ether_addr *addr)
 474 {
 475         struct pmd_internals *internals = dev->data->dev_private;
 476         struct ifreq ifr = { };
 477         int sockfd = internals->rx_queue[0].sockfd;
 478         int ret;
 479
 480         if (sockfd == -1) {
 481                 PMD_LOG(ERR, "receive socket not found");
 482                 return -EINVAL;
 483         }
 484
 485         strlcpy(ifr.ifr_name, internals->if_name, IFNAMSIZ);
 486         ifr.ifr_hwaddr.sa_family = ARPHRD_ETHER;
 487         memcpy(ifr.ifr_hwaddr.sa_data, addr, sizeof(*addr));
 488         ret = ioctl(sockfd, SIOCSIFHWADDR, &ifr);
 489
 490         if (ret < 0) {
 491                 PMD_LOG_ERRNO(ERR, "ioctl(SIOCSIFHWADDR) failed");
 492                 return -EINVAL;
 493         }
 494
 495         return 0;
 496 }
 497
 498 static int
 499 eth_dev_change_flags(char *if_name, uint32_t flags, uint32_t mask)
 500 {
 501         struct ifreq ifr;
 502         int ret = 0;
 503         int s;
 504
 505         s = socket(PF_INET, SOCK_DGRAM, 0);
 506         if (s < 0)
 507                 return -errno;
 508
 509         strlcpy(ifr.ifr_name, if_name, IFNAMSIZ);
 510         if (ioctl(s, SIOCGIFFLAGS, &ifr) < 0) {
 511                 ret = -errno;
 512                 goto out;
 513         }
 514         ifr.ifr_flags &= mask;
 515         ifr.ifr_flags |= flags;
 516         if (ioctl(s, SIOCSIFFLAGS, &ifr) < 0) {
 517                 ret = -errno;
 518                 goto out;
 519         }
 520 out:
 521         close(s);
 522         return ret;
 523 }
 524
 525 static int
 526 eth_dev_promiscuous_enable(struct rte_eth_dev *dev)
 527 {
 528         struct pmd_internals *internals = dev->data->dev_private;
 529
 530         return eth_dev_change_flags(internals->if_name, IFF_PROMISC, ~0);
 531 }
 532
 533 static int
 534 eth_dev_promiscuous_disable(struct rte_eth_dev *dev)
 535 {
 536         struct pmd_internals *internals = dev->data->dev_private;
 537
 538         return eth_dev_change_flags(internals->if_name, 0, ~IFF_PROMISC);
 539 }
 540
 541 static const struct eth_dev_ops ops = {
 542         .dev_start = eth_dev_start,
 543         .dev_stop = eth_dev_stop,
 544         .dev_close = eth_dev_close,
 545         .dev_configure = eth_dev_configure,
 546         .dev_infos_get = eth_dev_info,
 547         .mac_addr_set = eth_dev_macaddr_set,
 548         .mtu_set = eth_dev_mtu_set,
 549         .promiscuous_enable = eth_dev_promiscuous_enable,
 550         .promiscuous_disable = eth_dev_promiscuous_disable,
 551         .rx_queue_setup = eth_rx_queue_setup,
 552         .tx_queue_setup = eth_tx_queue_setup,
 553         .rx_queue_release = eth_queue_release,
 554         .tx_queue_release = eth_queue_release,
 555         .link_update = eth_link_update,
 556         .stats_get = eth_stats_get,
 557         .stats_reset = eth_stats_reset,
 558 };
 559
 560 /*
 561  * Opens an AF_PACKET socket
 562  */
 563 static int
 564 open_packet_iface(const char *key __rte_unused,
 565                   const char *value __rte_unused,
 566                   void *extra_args)
 567 {
 568         int *sockfd = extra_args;
 569
 570         /* Open an AF_PACKET socket... */
 571         *sockfd = socket(AF_PACKET, SOCK_RAW, htons(ETH_P_ALL));
 572         if (*sockfd == -1) {
 573                 PMD_LOG(ERR, "Could not open AF_PACKET socket");
 574                 return -1;
 575         }
 576
 577         return 0;
 578 }
 579
 580 static int
 581 rte_pmd_init_internals(struct rte_vdev_device *dev,
 582                        const int sockfd,
 583                        const unsigned nb_queues,
 584                        unsigned int blocksize,
 585                        unsigned int blockcnt,
 586                        unsigned int framesize,
 587                        unsigned int framecnt,
 588                        unsigned int qdisc_bypass,
 589                        struct pmd_internals **internals,
 590                        struct rte_eth_dev **eth_dev,
 591                        struct rte_kvargs *kvlist)
 592 {
 593         const char *name = rte_vdev_device_name(dev);
 594         const unsigned int numa_node = dev->device.numa_node;
 595         struct rte_eth_dev_data *data = NULL;
 596         struct rte_kvargs_pair *pair = NULL;
 597         struct ifreq ifr;
 598         size_t ifnamelen;
 599         unsigned k_idx;
 600         struct sockaddr_ll sockaddr;
 601         struct tpacket_req *req;
 602         struct pkt_rx_queue *rx_queue;
 603         struct pkt_tx_queue *tx_queue;
 604         int rc, tpver, discard;
 605         int qsockfd = -1;
 606         unsigned int i, q, rdsize;
 607 #if defined(PACKET_FANOUT)
 608         int fanout_arg;
 609 #endif
 610
 611         for (k_idx = 0; k_idx < kvlist->count; k_idx++) {
 612                 pair = &kvlist->pairs[k_idx];
 613                 if (strstr(pair->key, ETH_AF_PACKET_IFACE_ARG) != NULL)
 614                         break;
 615         }
 616         if (pair == NULL) {
 617                 PMD_LOG(ERR,
 618                         "%s: no interface specified for AF_PACKET ethdev",
 619                         name);
 620                 return -1;
 621         }
 622
 623         PMD_LOG(INFO,
 624                 "%s: creating AF_PACKET-backed ethdev on numa socket %u",
 625                 name, numa_node);
 626
 627         *internals = rte_zmalloc_socket(name, sizeof(**internals),
 628                                         0, numa_node);
 629         if (*internals == NULL)
 630                 return -1;
 631
 632         for (q = 0; q < nb_queues; q++) {
 633                 (*internals)->rx_queue[q].map = MAP_FAILED;
 634                 (*internals)->tx_queue[q].map = MAP_FAILED;
 635         }
 636
 637         req = &((*internals)->req);
 638
 639         req->tp_block_size = blocksize;
 640         req->tp_block_nr = blockcnt;
 641         req->tp_frame_size = framesize;
 642         req->tp_frame_nr = framecnt;
 643
 644         ifnamelen = strlen(pair->value);
 645         if (ifnamelen < sizeof(ifr.ifr_name)) {
 646                 memcpy(ifr.ifr_name, pair->value, ifnamelen);
 647                 ifr.ifr_name[ifnamelen] = '\0';
 648         } else {
 649                 PMD_LOG(ERR,
 650                         "%s: I/F name too long (%s)",
 651                         name, pair->value);
 652                 return -1;
 653         }
 654         if (ioctl(sockfd, SIOCGIFINDEX, &ifr) == -1) {
 655                 PMD_LOG_ERRNO(ERR, "%s: ioctl failed (SIOCGIFINDEX)", name);
 656                 return -1;
 657         }
 658         (*internals)->if_name = strdup(pair->value);
 659         if ((*internals)->if_name == NULL)
 660                 return -1;
 661         (*internals)->if_index = ifr.ifr_ifindex;
 662
 663         if (ioctl(sockfd, SIOCGIFHWADDR, &ifr) == -1) {
 664                 PMD_LOG_ERRNO(ERR, "%s: ioctl failed (SIOCGIFHWADDR)", name);
 665                 return -1;
 666         }
 667         memcpy(&(*internals)->eth_addr, ifr.ifr_hwaddr.sa_data, ETH_ALEN);
 668
 669         memset(&sockaddr, 0, sizeof(sockaddr));
 670         sockaddr.sll_family = AF_PACKET;
 671         sockaddr.sll_protocol = htons(ETH_P_ALL);
 672         sockaddr.sll_ifindex = (*internals)->if_index;
 673
 674 #if defined(PACKET_FANOUT)
 675         fanout_arg = (getpid() ^ (*internals)->if_index) & 0xffff;
 676         fanout_arg |= (PACKET_FANOUT_HASH | PACKET_FANOUT_FLAG_DEFRAG) << 16;
 677 #if defined(PACKET_FANOUT_FLAG_ROLLOVER)
 678         fanout_arg |= PACKET_FANOUT_FLAG_ROLLOVER << 16;
 679 #endif
 680 #endif
 681
 682         for (q = 0; q < nb_queues; q++) {
 683                 /* Open an AF_PACKET socket for this queue... */
 684                 qsockfd = socket(AF_PACKET, SOCK_RAW, htons(ETH_P_ALL));
 685                 if (qsockfd == -1) {
 686                         PMD_LOG_ERRNO(ERR,
 687                                 "%s: could not open AF_PACKET socket",
 688                                 name);
 689                         return -1;
 690                 }
 691
 692                 tpver = TPACKET_V2;
 693                 rc = setsockopt(qsockfd, SOL_PACKET, PACKET_VERSION,
 694                                 &tpver, sizeof(tpver));
 695                 if (rc == -1) {
 696                         PMD_LOG_ERRNO(ERR,
 697                                 "%s: could not set PACKET_VERSION on AF_PACKET socket for %s",
 698                                 name, pair->value);
 699                         goto error;
 700                 }
 701
 702                 discard = 1;
 703                 rc = setsockopt(qsockfd, SOL_PACKET, PACKET_LOSS,
 704                                 &discard, sizeof(discard));
 705                 if (rc == -1) {
 706                         PMD_LOG_ERRNO(ERR,
 707                                 "%s: could not set PACKET_LOSS on AF_PACKET socket for %s",
 708                                 name, pair->value);
 709                         goto error;
 710                 }
 711
 712 #if defined(PACKET_QDISC_BYPASS)
 713                 rc = setsockopt(qsockfd, SOL_PACKET, PACKET_QDISC_BYPASS,
 714                                 &qdisc_bypass, sizeof(qdisc_bypass));
 715                 if (rc == -1) {
 716                         PMD_LOG_ERRNO(ERR,
 717                                 "%s: could not set PACKET_QDISC_BYPASS on AF_PACKET socket for %s",
 718                                 name, pair->value);
 719                         goto error;
 720                 }
 721 #else
 722                 RTE_SET_USED(qdisc_bypass);
 723 #endif
 724
 725                 rc = setsockopt(qsockfd, SOL_PACKET, PACKET_RX_RING, req, sizeof(*req));
 726                 if (rc == -1) {
 727                         PMD_LOG_ERRNO(ERR,
 728                                 "%s: could not set PACKET_RX_RING on AF_PACKET socket for %s",
 729                                 name, pair->value);
 730                         goto error;
 731                 }
 732
 733                 rc = setsockopt(qsockfd, SOL_PACKET, PACKET_TX_RING, req, sizeof(*req));
 734                 if (rc == -1) {
 735                         PMD_LOG_ERRNO(ERR,
 736                                 "%s: could not set PACKET_TX_RING on AF_PACKET "
 737                                 "socket for %s", name, pair->value);
 738                         goto error;
 739                 }
 740
 741                 rx_queue = &((*internals)->rx_queue[q]);
 742                 rx_queue->framecount = req->tp_frame_nr;
 743
 744                 rx_queue->map = mmap(NULL, 2 * req->tp_block_size * req->tp_block_nr,
 745                                     PROT_READ | PROT_WRITE, MAP_SHARED | MAP_LOCKED,
 746                                     qsockfd, 0);
 747                 if (rx_queue->map == MAP_FAILED) {
 748                         PMD_LOG_ERRNO(ERR,
 749                                 "%s: call to mmap failed on AF_PACKET socket for %s",
 750                                 name, pair->value);
 751                         goto error;
 752                 }
 753
 754                 /* rdsize is same for both Tx and Rx */
 755                 rdsize = req->tp_frame_nr * sizeof(*(rx_queue->rd));
 756
 757                 rx_queue->rd = rte_zmalloc_socket(name, rdsize, 0, numa_node);
 758                 if (rx_queue->rd == NULL)
 759                         goto error;
 760                 for (i = 0; i < req->tp_frame_nr; ++i) {
 761                         rx_queue->rd[i].iov_base = rx_queue->map + (i * framesize);
 762                         rx_queue->rd[i].iov_len = req->tp_frame_size;
 763                 }
 764                 rx_queue->sockfd = qsockfd;
 765
 766                 tx_queue = &((*internals)->tx_queue[q]);
 767                 tx_queue->framecount = req->tp_frame_nr;
 768                 tx_queue->frame_data_size = req->tp_frame_size;
 769                 tx_queue->frame_data_size -= TPACKET2_HDRLEN -
 770                         sizeof(struct sockaddr_ll);
 771
 772                 tx_queue->map = rx_queue->map + req->tp_block_size * req->tp_block_nr;
 773
 774                 tx_queue->rd = rte_zmalloc_socket(name, rdsize, 0, numa_node);
 775                 if (tx_queue->rd == NULL)
 776                         goto error;
 777                 for (i = 0; i < req->tp_frame_nr; ++i) {
 778                         tx_queue->rd[i].iov_base = tx_queue->map + (i * framesize);
 779                         tx_queue->rd[i].iov_len = req->tp_frame_size;
 780                 }
 781                 tx_queue->sockfd = qsockfd;
 782
 783                 rc = bind(qsockfd, (const struct sockaddr*)&sockaddr, sizeof(sockaddr));
 784                 if (rc == -1) {
 785                         PMD_LOG_ERRNO(ERR,
 786                                 "%s: could not bind AF_PACKET socket to %s",
 787                                 name, pair->value);
 788                         goto error;
 789                 }
 790
 791 #if defined(PACKET_FANOUT)
 792                 rc = setsockopt(qsockfd, SOL_PACKET, PACKET_FANOUT,
 793                                 &fanout_arg, sizeof(fanout_arg));
 794                 if (rc == -1) {
 795                         PMD_LOG_ERRNO(ERR,
 796                                 "%s: could not set PACKET_FANOUT on AF_PACKET socket for %s",
 797                                 name, pair->value);
 798                         goto error;
 799                 }
 800 #endif
 801         }
 802
 803         /* reserve an ethdev entry */
 804         *eth_dev = rte_eth_vdev_allocate(dev, 0);
 805         if (*eth_dev == NULL)
 806                 goto error;
 807
 808         /*
 809          * now put it all together
 810          * - store queue data in internals,
 811          * - store numa_node in eth_dev
 812          * - point eth_dev_data to internals
 813          * - and point eth_dev structure to new eth_dev_data structure
 814          */
 815
 816         (*internals)->nb_queues = nb_queues;
 817
 818         data = (*eth_dev)->data;
 819         data->dev_private = *internals;
 820         data->nb_rx_queues = (uint16_t)nb_queues;
 821         data->nb_tx_queues = (uint16_t)nb_queues;
 822         data->dev_link = pmd_link;
 823         data->mac_addrs = &(*internals)->eth_addr;
 824
 825         (*eth_dev)->dev_ops = &ops;
 826
 827         return 0;
 828
 829 error:
 830         if (qsockfd != -1)
 831                 close(qsockfd);
 832         for (q = 0; q < nb_queues; q++) {
 833                 munmap((*internals)->rx_queue[q].map,
 834                        2 * req->tp_block_size * req->tp_block_nr);
 835
 836                 rte_free((*internals)->rx_queue[q].rd);
 837                 rte_free((*internals)->tx_queue[q].rd);
 838                 if (((*internals)->rx_queue[q].sockfd != 0) &&
 839                         ((*internals)->rx_queue[q].sockfd != qsockfd))
 840                         close((*internals)->rx_queue[q].sockfd);
 841         }
 842         free((*internals)->if_name);
 843         rte_free(*internals);
 844         return -1;
 845 }
 846
 847 static int
 848 rte_eth_from_packet(struct rte_vdev_device *dev,
 849                     int const *sockfd,
 850                     struct rte_kvargs *kvlist)
 851 {
 852         const char *name = rte_vdev_device_name(dev);
 853         struct pmd_internals *internals = NULL;
 854         struct rte_eth_dev *eth_dev = NULL;
 855         struct rte_kvargs_pair *pair = NULL;
 856         unsigned k_idx;
 857         unsigned int blockcount;
 858         unsigned int blocksize;
 859         unsigned int framesize = DFLT_FRAME_SIZE;
 860         unsigned int framecount = DFLT_FRAME_COUNT;
 861         unsigned int qpairs = 1;
 862         unsigned int qdisc_bypass = 1;
 863
 864         /* do some parameter checking */
 865         if (*sockfd < 0)
 866                 return -1;
 867
 868         blocksize = getpagesize();
 869
 870         /*
 871          * Walk arguments for configurable settings
 872          */
 873         for (k_idx = 0; k_idx < kvlist->count; k_idx++) {
 874                 pair = &kvlist->pairs[k_idx];
 875                 if (strstr(pair->key, ETH_AF_PACKET_NUM_Q_ARG) != NULL) {
 876                         qpairs = atoi(pair->value);
 877                         if (qpairs < 1 ||
 878                             qpairs > RTE_PMD_AF_PACKET_MAX_RINGS) {
 879                                 PMD_LOG(ERR,
 880                                         "%s: invalid qpairs value",
 881                                         name);
 882                                 return -1;
 883                         }
 884                         continue;
 885                 }
 886                 if (strstr(pair->key, ETH_AF_PACKET_BLOCKSIZE_ARG) != NULL) {
 887                         blocksize = atoi(pair->value);
 888                         if (!blocksize) {
 889                                 PMD_LOG(ERR,
 890                                         "%s: invalid blocksize value",
 891                                         name);
 892                                 return -1;
 893                         }
 894                         continue;
 895                 }
 896                 if (strstr(pair->key, ETH_AF_PACKET_FRAMESIZE_ARG) != NULL) {
 897                         framesize = atoi(pair->value);
 898                         if (!framesize) {
 899                                 PMD_LOG(ERR,
 900                                         "%s: invalid framesize value",
 901                                         name);
 902                                 return -1;
 903                         }
 904                         continue;
 905                 }
 906                 if (strstr(pair->key, ETH_AF_PACKET_FRAMECOUNT_ARG) != NULL) {
 907                         framecount = atoi(pair->value);
 908                         if (!framecount) {
 909                                 PMD_LOG(ERR,
 910                                         "%s: invalid framecount value",
 911                                         name);
 912                                 return -1;
 913                         }
 914                         continue;
 915                 }
 916                 if (strstr(pair->key, ETH_AF_PACKET_QDISC_BYPASS_ARG) != NULL) {
 917                         qdisc_bypass = atoi(pair->value);
 918                         if (qdisc_bypass > 1) {
 919                                 PMD_LOG(ERR,
 920                                         "%s: invalid bypass value",
 921                                         name);
 922                                 return -1;
 923                         }
 924                         continue;
 925                 }
 926         }
 927
 928         if (framesize > blocksize) {
 929                 PMD_LOG(ERR,
 930                         "%s: AF_PACKET MMAP frame size exceeds block size!",
 931                         name);
 932                 return -1;
 933         }
 934
 935         blockcount = framecount / (blocksize / framesize);
 936         if (!blockcount) {
 937                 PMD_LOG(ERR,
 938                         "%s: invalid AF_PACKET MMAP parameters", name);
 939                 return -1;
 940         }
 941
 942         PMD_LOG(INFO, "%s: AF_PACKET MMAP parameters:", name);
 943         PMD_LOG(INFO, "%s:\tblock size %d", name, blocksize);
 944         PMD_LOG(INFO, "%s:\tblock count %d", name, blockcount);
 945         PMD_LOG(INFO, "%s:\tframe size %d", name, framesize);
 946         PMD_LOG(INFO, "%s:\tframe count %d", name, framecount);
 947
 948         if (rte_pmd_init_internals(dev, *sockfd, qpairs,
 949                                    blocksize, blockcount,
 950                                    framesize, framecount,
 951                                    qdisc_bypass,
 952                                    &internals, &eth_dev,
 953                                    kvlist) < 0)
 954                 return -1;
 955
 956         eth_dev->rx_pkt_burst = eth_af_packet_rx;
 957         eth_dev->tx_pkt_burst = eth_af_packet_tx;
 958
 959         rte_eth_dev_probing_finish(eth_dev);
 960         return 0;
 961 }
 962
 963 static int
 964 rte_pmd_af_packet_probe(struct rte_vdev_device *dev)
 965 {
 966         int ret = 0;
 967         struct rte_kvargs *kvlist;
 968         int sockfd = -1;
 969         struct rte_eth_dev *eth_dev;
 970         const char *name = rte_vdev_device_name(dev);
 971
 972         PMD_LOG(INFO, "Initializing pmd_af_packet for %s", name);
 973
 974         if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
 975                 eth_dev = rte_eth_dev_attach_secondary(name);
 976                 if (!eth_dev) {
 977                         PMD_LOG(ERR, "Failed to probe %s", name);
 978                         return -1;
 979                 }
 980                 /* TODO: request info from primary to set up Rx and Tx */
 981                 eth_dev->dev_ops = &ops;
 982                 eth_dev->device = &dev->device;
 983                 rte_eth_dev_probing_finish(eth_dev);
 984                 return 0;
 985         }
 986
 987         kvlist = rte_kvargs_parse(rte_vdev_device_args(dev), valid_arguments);
 988         if (kvlist == NULL) {
 989                 ret = -1;
 990                 goto exit;
 991         }
 992
 993         /*
 994          * If iface argument is passed we open the NICs and use them for
 995          * reading / writing
 996          */
 997         if (rte_kvargs_count(kvlist, ETH_AF_PACKET_IFACE_ARG) == 1) {
 998
 999                 ret = rte_kvargs_process(kvlist, ETH_AF_PACKET_IFACE_ARG,
1000                                          &open_packet_iface, &sockfd);
1001                 if (ret < 0)
1002                         goto exit;
1003         }
1004
1005         if (dev->device.numa_node == SOCKET_ID_ANY)
1006                 dev->device.numa_node = rte_socket_id();
1007
1008         ret = rte_eth_from_packet(dev, &sockfd, kvlist);
1009         close(sockfd); /* no longer needed */
1010
1011 exit:
1012         rte_kvargs_free(kvlist);
1013         return ret;
1014 }
1015
1016 static int
1017 rte_pmd_af_packet_remove(struct rte_vdev_device *dev)
1018 {
1019         struct rte_eth_dev *eth_dev = NULL;
1020         struct pmd_internals *internals;
1021         struct tpacket_req *req;
1022         unsigned q;
1023
1024         PMD_LOG(INFO, "Closing AF_PACKET ethdev on numa socket %u",
1025                 rte_socket_id());
1026
1027         if (dev == NULL)
1028                 return -1;
1029
1030         /* find the ethdev entry */
1031         eth_dev = rte_eth_dev_allocated(rte_vdev_device_name(dev));
1032         if (eth_dev == NULL)
1033                 return -1;
1034
1035         /* mac_addrs must not be freed alone because part of dev_private */
1036         eth_dev->data->mac_addrs = NULL;
1037
1038         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1039                 return rte_eth_dev_release_port(eth_dev);
1040
1041         internals = eth_dev->data->dev_private;
1042         req = &internals->req;
1043         for (q = 0; q < internals->nb_queues; q++) {
1044                 munmap(internals->rx_queue[q].map,
1045                         2 * req->tp_block_size * req->tp_block_nr);
1046                 rte_free(internals->rx_queue[q].rd);
1047                 rte_free(internals->tx_queue[q].rd);
1048         }
1049         free(internals->if_name);
1050
1051         rte_eth_dev_release_port(eth_dev);
1052
1053         return 0;
1054 }
1055
1056 static struct rte_vdev_driver pmd_af_packet_drv = {
1057         .probe = rte_pmd_af_packet_probe,
1058         .remove = rte_pmd_af_packet_remove,
1059 };
1060
1061 RTE_PMD_REGISTER_VDEV(net_af_packet, pmd_af_packet_drv);
1062 RTE_PMD_REGISTER_ALIAS(net_af_packet, eth_af_packet);
1063 RTE_PMD_REGISTER_PARAM_STRING(net_af_packet,
1064         "iface=<string> "
1065         "qpairs=<int> "
1066         "blocksz=<int> "
1067         "framesz=<int> "
1068         "framecnt=<int> "
1069         "qdisc_bypass=<0|1>");
1070
1071 RTE_INIT(af_packet_init_log)
1072 {
1073         af_packet_logtype = rte_log_register("pmd.net.packet");
1074         if (af_packet_logtype >= 0)
1075                 rte_log_set_level(af_packet_logtype, RTE_LOG_NOTICE);
1076 }