Lines Matching refs:cfv
142 struct cfv_info *cfv = vq_tx->vdev->priv; in cfv_release_cb() local
144 ++cfv->stats.tx_kicks; in cfv_release_cb()
145 tasklet_schedule(&cfv->tx_release_tasklet); in cfv_release_cb()
148 static void free_buf_info(struct cfv_info *cfv, struct buf_info *buf_info) in free_buf_info() argument
152 gen_pool_free(cfv->genpool, (unsigned long) buf_info->vaddr, in free_buf_info()
162 struct cfv_info *cfv = vq_tx->vdev->priv; in cfv_release_used_buf() local
165 BUG_ON(vq_tx != cfv->vq_tx); in cfv_release_used_buf()
172 spin_lock_irqsave(&cfv->tx_lock, flags); in cfv_release_used_buf()
174 spin_unlock_irqrestore(&cfv->tx_lock, flags); in cfv_release_used_buf()
180 free_buf_info(cfv, buf_info); in cfv_release_used_buf()
186 if (cfv->vq_tx->num_free <= cfv->watermark_tx) in cfv_release_used_buf()
190 if (cfv->reserved_mem == 0 && cfv->genpool) in cfv_release_used_buf()
191 cfv->reserved_mem = in cfv_release_used_buf()
192 gen_pool_alloc(cfv->genpool, in cfv_release_used_buf()
193 cfv->reserved_size); in cfv_release_used_buf()
196 if (cfv->reserved_mem) { in cfv_release_used_buf()
197 cfv->watermark_tx = in cfv_release_used_buf()
198 virtqueue_get_vring_size(cfv->vq_tx); in cfv_release_used_buf()
199 netif_tx_wake_all_queues(cfv->ndev); in cfv_release_used_buf()
203 virtqueue_disable_cb(cfv->vq_tx); in cfv_release_used_buf()
204 ++cfv->stats.tx_flow_on; in cfv_release_used_buf()
207 WARN_ON(cfv->watermark_tx > in cfv_release_used_buf()
208 virtqueue_get_vring_size(cfv->vq_tx)); in cfv_release_used_buf()
209 cfv->watermark_tx += in cfv_release_used_buf()
210 virtqueue_get_vring_size(cfv->vq_tx) / 4; in cfv_release_used_buf()
217 struct cfv_info *cfv, in cfv_alloc_and_copy_skb() argument
225 if (frm_len > cfv->mru || frm_len <= cfv->rx_hr + cfv->rx_tr) { in cfv_alloc_and_copy_skb()
226 netdev_err(cfv->ndev, in cfv_alloc_and_copy_skb()
228 frm_len, cfv->mru, cfv->rx_hr, in cfv_alloc_and_copy_skb()
229 cfv->rx_tr); in cfv_alloc_and_copy_skb()
234 cfpkt_len = frm_len - (cfv->rx_hr + cfv->rx_tr); in cfv_alloc_and_copy_skb()
235 pad_len = (unsigned long)(frm + cfv->rx_hr) & (IP_HDR_ALIGN - 1); in cfv_alloc_and_copy_skb()
237 skb = netdev_alloc_skb(cfv->ndev, frm_len + pad_len); in cfv_alloc_and_copy_skb()
243 skb_reserve(skb, cfv->rx_hr + pad_len); in cfv_alloc_and_copy_skb()
245 memcpy(skb_put(skb, cfpkt_len), frm + cfv->rx_hr, cfpkt_len); in cfv_alloc_and_copy_skb()
252 struct cfv_info *cfv = container_of(napi, struct cfv_info, napi); in cfv_rx_poll() local
257 struct vringh_kiov *riov = &cfv->ctx.riov; in cfv_rx_poll()
267 if (cfv->ctx.head != USHRT_MAX) { in cfv_rx_poll()
268 vringh_complete_kern(cfv->vr_rx, in cfv_rx_poll()
269 cfv->ctx.head, in cfv_rx_poll()
271 cfv->ctx.head = USHRT_MAX; in cfv_rx_poll()
275 cfv->vr_rx, in cfv_rx_poll()
278 &cfv->ctx.head, in cfv_rx_poll()
288 skb = cfv_alloc_and_copy_skb(&err, cfv, buf, in cfv_rx_poll()
297 skb->dev = cfv->ndev; in cfv_rx_poll()
300 ++cfv->ndev->stats.rx_dropped; in cfv_rx_poll()
302 ++cfv->ndev->stats.rx_packets; in cfv_rx_poll()
303 cfv->ndev->stats.rx_bytes += skb_len; in cfv_rx_poll()
310 ++cfv->stats.rx_napi_resched; in cfv_rx_poll()
316 ++cfv->stats.rx_napi_complete; in cfv_rx_poll()
320 if (unlikely(!vringh_notify_enable_kern(cfv->vr_rx)) && in cfv_rx_poll()
322 vringh_notify_disable_kern(cfv->vr_rx); in cfv_rx_poll()
328 ++cfv->stats.rx_nomem; in cfv_rx_poll()
332 vringh_notify_enable_kern(cfv->vr_rx); in cfv_rx_poll()
337 netdev_warn(cfv->ndev, "Bad ring, disable device\n"); in cfv_rx_poll()
338 cfv->ndev->stats.rx_dropped = riov->used - riov->i; in cfv_rx_poll()
340 vringh_notify_disable_kern(cfv->vr_rx); in cfv_rx_poll()
341 netif_carrier_off(cfv->ndev); in cfv_rx_poll()
345 if (rxcnt && vringh_need_notify_kern(cfv->vr_rx) > 0) in cfv_rx_poll()
346 vringh_notify(cfv->vr_rx); in cfv_rx_poll()
352 struct cfv_info *cfv = vdev->priv; in cfv_recv() local
354 ++cfv->stats.rx_kicks; in cfv_recv()
355 vringh_notify_disable_kern(cfv->vr_rx); in cfv_recv()
356 napi_schedule(&cfv->napi); in cfv_recv()
359 static void cfv_destroy_genpool(struct cfv_info *cfv) in cfv_destroy_genpool() argument
361 if (cfv->alloc_addr) in cfv_destroy_genpool()
362 dma_free_coherent(cfv->vdev->dev.parent->parent, in cfv_destroy_genpool()
363 cfv->allocsz, cfv->alloc_addr, in cfv_destroy_genpool()
364 cfv->alloc_dma); in cfv_destroy_genpool()
366 if (!cfv->genpool) in cfv_destroy_genpool()
368 gen_pool_free(cfv->genpool, cfv->reserved_mem, in cfv_destroy_genpool()
369 cfv->reserved_size); in cfv_destroy_genpool()
370 gen_pool_destroy(cfv->genpool); in cfv_destroy_genpool()
371 cfv->genpool = NULL; in cfv_destroy_genpool()
374 static int cfv_create_genpool(struct cfv_info *cfv) in cfv_create_genpool() argument
384 cfv->allocsz = (virtqueue_get_vring_size(cfv->vq_tx) * in cfv_create_genpool()
385 (ETH_DATA_LEN + cfv->tx_hr + cfv->tx_tr) * 11)/10; in cfv_create_genpool()
386 if (cfv->allocsz <= (num_possible_cpus() + 1) * cfv->ndev->mtu) in cfv_create_genpool()
390 if (cfv->allocsz <= num_possible_cpus() * cfv->ndev->mtu) { in cfv_create_genpool()
391 netdev_info(cfv->ndev, "Not enough device memory\n"); in cfv_create_genpool()
395 cfv->alloc_addr = dma_alloc_coherent( in cfv_create_genpool()
396 cfv->vdev->dev.parent->parent, in cfv_create_genpool()
397 cfv->allocsz, &cfv->alloc_dma, in cfv_create_genpool()
399 if (cfv->alloc_addr) in cfv_create_genpool()
402 cfv->allocsz = (cfv->allocsz * 3) >> 2; in cfv_create_genpool()
405 netdev_dbg(cfv->ndev, "Allocated %zd bytes from dma-memory\n", in cfv_create_genpool()
406 cfv->allocsz); in cfv_create_genpool()
409 cfv->genpool = gen_pool_create(7, -1); in cfv_create_genpool()
410 if (!cfv->genpool) in cfv_create_genpool()
413 err = gen_pool_add_virt(cfv->genpool, (unsigned long)cfv->alloc_addr, in cfv_create_genpool()
414 (phys_addr_t)virt_to_phys(cfv->alloc_addr), in cfv_create_genpool()
415 cfv->allocsz, -1); in cfv_create_genpool()
422 cfv->reserved_size = num_possible_cpus() * cfv->ndev->mtu; in cfv_create_genpool()
423 cfv->reserved_mem = gen_pool_alloc(cfv->genpool, in cfv_create_genpool()
424 cfv->reserved_size); in cfv_create_genpool()
425 if (!cfv->reserved_mem) { in cfv_create_genpool()
430 cfv->watermark_tx = virtqueue_get_vring_size(cfv->vq_tx); in cfv_create_genpool()
433 cfv_destroy_genpool(cfv); in cfv_create_genpool()
440 struct cfv_info *cfv = netdev_priv(netdev); in cfv_netdev_open() local
442 if (cfv_create_genpool(cfv)) in cfv_netdev_open()
446 napi_enable(&cfv->napi); in cfv_netdev_open()
449 napi_schedule(&cfv->napi); in cfv_netdev_open()
456 struct cfv_info *cfv = netdev_priv(netdev); in cfv_netdev_close() local
462 virtqueue_disable_cb(cfv->vq_tx); in cfv_netdev_close()
463 vringh_notify_disable_kern(cfv->vr_rx); in cfv_netdev_close()
464 napi_disable(&cfv->napi); in cfv_netdev_close()
467 cfv_release_used_buf(cfv->vq_tx); in cfv_netdev_close()
468 spin_lock_irqsave(&cfv->tx_lock, flags); in cfv_netdev_close()
469 while ((buf_info = virtqueue_detach_unused_buf(cfv->vq_tx))) in cfv_netdev_close()
470 free_buf_info(cfv, buf_info); in cfv_netdev_close()
471 spin_unlock_irqrestore(&cfv->tx_lock, flags); in cfv_netdev_close()
474 cfv_destroy_genpool(cfv); in cfv_netdev_close()
479 static struct buf_info *cfv_alloc_and_copy_to_shm(struct cfv_info *cfv, in cfv_alloc_and_copy_to_shm() argument
487 if (!cfv->genpool) in cfv_alloc_and_copy_to_shm()
490 if (unlikely(cfv->tx_hr + skb->len + cfv->tx_tr > cfv->mtu)) { in cfv_alloc_and_copy_to_shm()
491 netdev_warn(cfv->ndev, "Invalid packet len (%d > %d)\n", in cfv_alloc_and_copy_to_shm()
492 cfv->tx_hr + skb->len + cfv->tx_tr, cfv->mtu); in cfv_alloc_and_copy_to_shm()
501 hdr_ofs = cfv->tx_hr + info->hdr_len; in cfv_alloc_and_copy_to_shm()
503 buf_info->size = cfv->tx_hr + skb->len + cfv->tx_tr + pad_len; in cfv_alloc_and_copy_to_shm()
506 buf_info->vaddr = (void *)gen_pool_alloc(cfv->genpool, buf_info->size); in cfv_alloc_and_copy_to_shm()
511 skb_copy_bits(skb, 0, buf_info->vaddr + cfv->tx_hr + pad_len, skb->len); in cfv_alloc_and_copy_to_shm()
513 skb->len + cfv->tx_hr + cfv->rx_hr); in cfv_alloc_and_copy_to_shm()
524 struct cfv_info *cfv = netdev_priv(netdev); in cfv_netdev_tx() local
532 cfv_release_used_buf(cfv->vq_tx); in cfv_netdev_tx()
533 spin_lock_irqsave(&cfv->tx_lock, flags); in cfv_netdev_tx()
540 if (unlikely(cfv->vq_tx->num_free <= num_present_cpus())) { in cfv_netdev_tx()
542 cfv->stats.tx_full_ring++; in cfv_netdev_tx()
548 buf_info = cfv_alloc_and_copy_to_shm(cfv, skb, &sg); in cfv_netdev_tx()
550 cfv->stats.tx_no_mem++; in cfv_netdev_tx()
553 if (cfv->reserved_mem && cfv->genpool) { in cfv_netdev_tx()
554 gen_pool_free(cfv->genpool, cfv->reserved_mem, in cfv_netdev_tx()
555 cfv->reserved_size); in cfv_netdev_tx()
556 cfv->reserved_mem = 0; in cfv_netdev_tx()
557 buf_info = cfv_alloc_and_copy_to_shm(cfv, skb, &sg); in cfv_netdev_tx()
563 cfv->watermark_tx = virtqueue_get_vring_size(cfv->vq_tx) / 4; in cfv_netdev_tx()
565 virtqueue_enable_cb(cfv->vq_tx); in cfv_netdev_tx()
571 netdev_warn(cfv->ndev, "Out of gen_pool memory\n"); in cfv_netdev_tx()
575 ret = virtqueue_add_outbuf(cfv->vq_tx, &sg, 1, buf_info, GFP_ATOMIC); in cfv_netdev_tx()
578 netdev_warn(cfv->ndev, "Failed adding buffer to TX vring:%d\n", in cfv_netdev_tx()
584 cfv->ndev->stats.tx_packets++; in cfv_netdev_tx()
585 cfv->ndev->stats.tx_bytes += skb->len; in cfv_netdev_tx()
586 spin_unlock_irqrestore(&cfv->tx_lock, flags); in cfv_netdev_tx()
589 virtqueue_kick(cfv->vq_tx); in cfv_netdev_tx()
594 spin_unlock_irqrestore(&cfv->tx_lock, flags); in cfv_netdev_tx()
595 cfv->ndev->stats.tx_dropped++; in cfv_netdev_tx()
596 free_buf_info(cfv, buf_info); in cfv_netdev_tx()
603 struct cfv_info *cfv = (struct cfv_info *)drv; in cfv_tx_release_tasklet() local
604 cfv_release_used_buf(cfv->vq_tx); in cfv_tx_release_tasklet()
624 static inline void debugfs_init(struct cfv_info *cfv) in debugfs_init() argument
626 cfv->debugfs = in debugfs_init()
627 debugfs_create_dir(netdev_name(cfv->ndev), NULL); in debugfs_init()
629 if (IS_ERR(cfv->debugfs)) in debugfs_init()
632 debugfs_create_u32("rx-napi-complete", S_IRUSR, cfv->debugfs, in debugfs_init()
633 &cfv->stats.rx_napi_complete); in debugfs_init()
634 debugfs_create_u32("rx-napi-resched", S_IRUSR, cfv->debugfs, in debugfs_init()
635 &cfv->stats.rx_napi_resched); in debugfs_init()
636 debugfs_create_u32("rx-nomem", S_IRUSR, cfv->debugfs, in debugfs_init()
637 &cfv->stats.rx_nomem); in debugfs_init()
638 debugfs_create_u32("rx-kicks", S_IRUSR, cfv->debugfs, in debugfs_init()
639 &cfv->stats.rx_kicks); in debugfs_init()
640 debugfs_create_u32("tx-full-ring", S_IRUSR, cfv->debugfs, in debugfs_init()
641 &cfv->stats.tx_full_ring); in debugfs_init()
642 debugfs_create_u32("tx-no-mem", S_IRUSR, cfv->debugfs, in debugfs_init()
643 &cfv->stats.tx_no_mem); in debugfs_init()
644 debugfs_create_u32("tx-kicks", S_IRUSR, cfv->debugfs, in debugfs_init()
645 &cfv->stats.tx_kicks); in debugfs_init()
646 debugfs_create_u32("tx-flow-on", S_IRUSR, cfv->debugfs, in debugfs_init()
647 &cfv->stats.tx_flow_on); in debugfs_init()
658 struct cfv_info *cfv; in cfv_probe() local
666 cfv = netdev_priv(netdev); in cfv_probe()
667 cfv->vdev = vdev; in cfv_probe()
668 cfv->ndev = netdev; in cfv_probe()
670 spin_lock_init(&cfv->tx_lock); in cfv_probe()
677 err = vdev->vringh_config->find_vrhs(vdev, 1, &cfv->vr_rx, &vrh_cbs); in cfv_probe()
682 err = vdev->config->find_vqs(vdev, 1, &cfv->vq_tx, &vq_cbs, &names); in cfv_probe()
689 &cfv->tx_hr); in cfv_probe()
691 &cfv->rx_hr); in cfv_probe()
693 &cfv->tx_tr); in cfv_probe()
695 &cfv->rx_tr); in cfv_probe()
697 &cfv->mtu); in cfv_probe()
699 &cfv->mru); in cfv_probe()
701 cfv->tx_hr = CFV_DEF_HEADROOM; in cfv_probe()
702 cfv->rx_hr = CFV_DEF_HEADROOM; in cfv_probe()
703 cfv->tx_tr = CFV_DEF_TAILROOM; in cfv_probe()
704 cfv->rx_tr = CFV_DEF_TAILROOM; in cfv_probe()
705 cfv->mtu = CFV_DEF_MTU_SIZE; in cfv_probe()
706 cfv->mru = CFV_DEF_MTU_SIZE; in cfv_probe()
709 netdev->needed_headroom = cfv->tx_hr; in cfv_probe()
710 netdev->needed_tailroom = cfv->tx_tr; in cfv_probe()
713 virtqueue_disable_cb(cfv->vq_tx); in cfv_probe()
715 netdev->mtu = cfv->mtu - cfv->tx_tr; in cfv_probe()
716 vdev->priv = cfv; in cfv_probe()
719 vringh_kiov_init(&cfv->ctx.riov, NULL, 0); in cfv_probe()
720 cfv->ctx.head = USHRT_MAX; in cfv_probe()
721 netif_napi_add(netdev, &cfv->napi, cfv_rx_poll, CFV_DEFAULT_QUOTA); in cfv_probe()
723 tasklet_init(&cfv->tx_release_tasklet, in cfv_probe()
725 (unsigned long)cfv); in cfv_probe()
737 debugfs_init(cfv); in cfv_probe()
741 netdev_warn(cfv->ndev, "CAIF Virtio probe failed:%d\n", err); in cfv_probe()
743 if (cfv->vr_rx) in cfv_probe()
744 vdev->vringh_config->del_vrhs(cfv->vdev); in cfv_probe()
745 if (cfv->vdev) in cfv_probe()
746 vdev->config->del_vqs(cfv->vdev); in cfv_probe()
753 struct cfv_info *cfv = vdev->priv; in cfv_remove() local
756 dev_close(cfv->ndev); in cfv_remove()
759 tasklet_kill(&cfv->tx_release_tasklet); in cfv_remove()
760 debugfs_remove_recursive(cfv->debugfs); in cfv_remove()
762 vringh_kiov_cleanup(&cfv->ctx.riov); in cfv_remove()
764 vdev->vringh_config->del_vrhs(cfv->vdev); in cfv_remove()
765 cfv->vr_rx = NULL; in cfv_remove()
766 vdev->config->del_vqs(cfv->vdev); in cfv_remove()
767 unregister_netdev(cfv->ndev); in cfv_remove()