385 	struct netlink_ring *ring;  in __netlink_set_ring()  local
388 	ring  = tx_ring ? &nlk->tx_ring : &nlk->rx_ring;  in __netlink_set_ring()
392 	ring->frame_max		= req->nm_frame_nr - 1;  in __netlink_set_ring()
393 	ring->head		= 0;  in __netlink_set_ring()
394 	ring->frame_size	= req->nm_frame_size;  in __netlink_set_ring()
395 	ring->pg_vec_pages	= req->nm_block_size / PAGE_SIZE;  in __netlink_set_ring()
397 	swap(ring->pg_vec_len, req->nm_block_nr);  in __netlink_set_ring()
398 	swap(ring->pg_vec_order, order);  in __netlink_set_ring()
399 	swap(ring->pg_vec, pg_vec);  in __netlink_set_ring()
414 	struct netlink_ring *ring;  in netlink_set_ring()  local
418 	ring  = tx_ring ? &nlk->tx_ring : &nlk->rx_ring;  in netlink_set_ring()
422 	if (atomic_read(&ring->pending))  in netlink_set_ring()
426 		if (ring->pg_vec != NULL)  in netlink_set_ring()
438 		ring->frames_per_block = req->nm_block_size /  in netlink_set_ring()
440 		if (ring->frames_per_block == 0)  in netlink_set_ring()
442 		if (ring->frames_per_block * req->nm_block_nr !=  in netlink_set_ring()
500 	struct netlink_ring *ring;  in netlink_mmap()  local
511 	for (ring = &nlk->rx_ring; ring <= &nlk->tx_ring; ring++) {  in netlink_mmap()
512 		if (ring->pg_vec == NULL)  in netlink_mmap()
514 		expected += ring->pg_vec_len * ring->pg_vec_pages * PAGE_SIZE;  in netlink_mmap()
525 	for (ring = &nlk->rx_ring; ring <= &nlk->tx_ring; ring++) {  in netlink_mmap()
526 		if (ring->pg_vec == NULL)  in netlink_mmap()
529 		for (i = 0; i < ring->pg_vec_len; i++) {  in netlink_mmap()
531 			void *kaddr = ring->pg_vec[i];  in netlink_mmap()
534 			for (pg_num = 0; pg_num < ring->pg_vec_pages; pg_num++) {  in netlink_mmap()
584 __netlink_lookup_frame(const struct netlink_ring *ring, unsigned int pos)  in __netlink_lookup_frame()  argument
588 	pg_vec_pos = pos / ring->frames_per_block;  in __netlink_lookup_frame()
589 	frame_off  = pos % ring->frames_per_block;  in __netlink_lookup_frame()
591 	return ring->pg_vec[pg_vec_pos] + (frame_off * ring->frame_size);  in __netlink_lookup_frame()
595 netlink_lookup_frame(const struct netlink_ring *ring, unsigned int pos,  in netlink_lookup_frame()  argument
600 	hdr = __netlink_lookup_frame(ring, pos);  in netlink_lookup_frame()
608 netlink_current_frame(const struct netlink_ring *ring,  in netlink_current_frame()  argument
611 	return netlink_lookup_frame(ring, ring->head, status);  in netlink_current_frame()
614 static void netlink_increment_head(struct netlink_ring *ring)  in netlink_increment_head()  argument
616 	ring->head = ring->head != ring->frame_max ? ring->head + 1 : 0;  in netlink_increment_head()
619 static void netlink_forward_ring(struct netlink_ring *ring)  in netlink_forward_ring()  argument
621 	unsigned int head = ring->head;  in netlink_forward_ring()
625 		hdr = __netlink_lookup_frame(ring, ring->head);  in netlink_forward_ring()
630 		netlink_increment_head(ring);  in netlink_forward_ring()
631 	} while (ring->head != head);  in netlink_forward_ring()
634 static bool netlink_has_valid_frame(struct netlink_ring *ring)  in netlink_has_valid_frame()  argument
636 	unsigned int head = ring->head, pos = head;  in netlink_has_valid_frame()
640 		hdr = __netlink_lookup_frame(ring, pos);  in netlink_has_valid_frame()
643 		pos = pos != 0 ? pos - 1 : ring->frame_max;  in netlink_has_valid_frame()
651 	struct netlink_ring *ring = &nlk->rx_ring;  in netlink_dump_space()  local
655 	hdr = netlink_current_frame(ring, NL_MMAP_STATUS_UNUSED);  in netlink_dump_space()
659 	n = ring->head + ring->frame_max / 2;  in netlink_dump_space()
660 	if (n > ring->frame_max)  in netlink_dump_space()
661 		n -= ring->frame_max;  in netlink_dump_space()
663 	hdr = __netlink_lookup_frame(ring, n);  in netlink_dump_space()
724 				   struct netlink_ring *ring,  in netlink_ring_setup_skb()  argument
730 	size = ring->frame_size - NL_MMAP_HDRLEN;  in netlink_ring_setup_skb()
749 	struct netlink_ring *ring;  in netlink_mmap_sendmsg()  local
757 	ring   = &nlk->tx_ring;  in netlink_mmap_sendmsg()
758 	maxlen = ring->frame_size - NL_MMAP_HDRLEN;  in netlink_mmap_sendmsg()
763 		hdr = netlink_current_frame(ring, NL_MMAP_STATUS_VALID);  in netlink_mmap_sendmsg()
788 		netlink_increment_head(ring);  in netlink_mmap_sendmsg()
842 	struct netlink_ring *ring = &nlk->rx_ring;  in netlink_ring_set_copied()  local
846 	hdr = netlink_current_frame(ring, NL_MMAP_STATUS_UNUSED);  in netlink_ring_set_copied()
853 	netlink_increment_head(ring);  in netlink_ring_set_copied()
877 	struct netlink_ring *ring;  in netlink_skb_destructor()  local
891 			ring = &nlk_sk(sk)->tx_ring;  in netlink_skb_destructor()
897 			ring = &nlk_sk(sk)->rx_ring;  in netlink_skb_destructor()
900 		WARN_ON(atomic_read(&ring->pending) == 0);  in netlink_skb_destructor()
901 		atomic_dec(&ring->pending);  in netlink_skb_destructor()
1887 	struct netlink_ring *ring;  in __netlink_alloc_skb()  local
1894 	ring = &nlk_sk(sk)->rx_ring;  in __netlink_alloc_skb()
1896 	if (ring->pg_vec == NULL)  in __netlink_alloc_skb()
1903 	if (ring->frame_size - NL_MMAP_HDRLEN < linear_size)  in __netlink_alloc_skb()
1912 	if (ring->pg_vec == NULL)  in __netlink_alloc_skb()
1916 	maxlen = ring->frame_size - NL_MMAP_HDRLEN;  in __netlink_alloc_skb()
1920 	netlink_forward_ring(ring);  in __netlink_alloc_skb()
1921 	hdr = netlink_current_frame(ring, NL_MMAP_STATUS_UNUSED);  in __netlink_alloc_skb()
1925 	netlink_ring_setup_skb(skb, sk, ring, hdr);  in __netlink_alloc_skb()
1927 	atomic_inc(&ring->pending);  in __netlink_alloc_skb()
1928 	netlink_increment_head(ring);  in __netlink_alloc_skb()