382 	struct netlink_ring *ring;  in __netlink_set_ring()  local
385 	ring  = tx_ring ? &nlk->tx_ring : &nlk->rx_ring;  in __netlink_set_ring()
389 	ring->frame_max		= req->nm_frame_nr - 1;  in __netlink_set_ring()
390 	ring->head		= 0;  in __netlink_set_ring()
391 	ring->frame_size	= req->nm_frame_size;  in __netlink_set_ring()
392 	ring->pg_vec_pages	= req->nm_block_size / PAGE_SIZE;  in __netlink_set_ring()
394 	swap(ring->pg_vec_len, req->nm_block_nr);  in __netlink_set_ring()
395 	swap(ring->pg_vec_order, order);  in __netlink_set_ring()
396 	swap(ring->pg_vec, pg_vec);  in __netlink_set_ring()
411 	struct netlink_ring *ring;  in netlink_set_ring()  local
415 	ring  = tx_ring ? &nlk->tx_ring : &nlk->rx_ring;  in netlink_set_ring()
419 	if (atomic_read(&ring->pending))  in netlink_set_ring()
423 		if (ring->pg_vec != NULL)  in netlink_set_ring()
435 		ring->frames_per_block = req->nm_block_size /  in netlink_set_ring()
437 		if (ring->frames_per_block == 0)  in netlink_set_ring()
439 		if (ring->frames_per_block * req->nm_block_nr !=  in netlink_set_ring()
497 	struct netlink_ring *ring;  in netlink_mmap()  local
508 	for (ring = &nlk->rx_ring; ring <= &nlk->tx_ring; ring++) {  in netlink_mmap()
509 		if (ring->pg_vec == NULL)  in netlink_mmap()
511 		expected += ring->pg_vec_len * ring->pg_vec_pages * PAGE_SIZE;  in netlink_mmap()
522 	for (ring = &nlk->rx_ring; ring <= &nlk->tx_ring; ring++) {  in netlink_mmap()
523 		if (ring->pg_vec == NULL)  in netlink_mmap()
526 		for (i = 0; i < ring->pg_vec_len; i++) {  in netlink_mmap()
528 			void *kaddr = ring->pg_vec[i];  in netlink_mmap()
531 			for (pg_num = 0; pg_num < ring->pg_vec_pages; pg_num++) {  in netlink_mmap()
581 __netlink_lookup_frame(const struct netlink_ring *ring, unsigned int pos)  in __netlink_lookup_frame()  argument
585 	pg_vec_pos = pos / ring->frames_per_block;  in __netlink_lookup_frame()
586 	frame_off  = pos % ring->frames_per_block;  in __netlink_lookup_frame()
588 	return ring->pg_vec[pg_vec_pos] + (frame_off * ring->frame_size);  in __netlink_lookup_frame()
592 netlink_lookup_frame(const struct netlink_ring *ring, unsigned int pos,  in netlink_lookup_frame()  argument
597 	hdr = __netlink_lookup_frame(ring, pos);  in netlink_lookup_frame()
605 netlink_current_frame(const struct netlink_ring *ring,  in netlink_current_frame()  argument
608 	return netlink_lookup_frame(ring, ring->head, status);  in netlink_current_frame()
612 netlink_previous_frame(const struct netlink_ring *ring,  in netlink_previous_frame()  argument
617 	prev = ring->head ? ring->head - 1 : ring->frame_max;  in netlink_previous_frame()
618 	return netlink_lookup_frame(ring, prev, status);  in netlink_previous_frame()
621 static void netlink_increment_head(struct netlink_ring *ring)  in netlink_increment_head()  argument
623 	ring->head = ring->head != ring->frame_max ? ring->head + 1 : 0;  in netlink_increment_head()
626 static void netlink_forward_ring(struct netlink_ring *ring)  in netlink_forward_ring()  argument
628 	unsigned int head = ring->head, pos = head;  in netlink_forward_ring()
632 		hdr = __netlink_lookup_frame(ring, pos);  in netlink_forward_ring()
637 		netlink_increment_head(ring);  in netlink_forward_ring()
638 	} while (ring->head != head);  in netlink_forward_ring()
643 	struct netlink_ring *ring = &nlk->rx_ring;  in netlink_dump_space()  local
647 	hdr = netlink_current_frame(ring, NL_MMAP_STATUS_UNUSED);  in netlink_dump_space()
651 	n = ring->head + ring->frame_max / 2;  in netlink_dump_space()
652 	if (n > ring->frame_max)  in netlink_dump_space()
653 		n -= ring->frame_max;  in netlink_dump_space()
655 	hdr = __netlink_lookup_frame(ring, n);  in netlink_dump_space()
710 				   struct netlink_ring *ring,  in netlink_ring_setup_skb()  argument
716 	size = ring->frame_size - NL_MMAP_HDRLEN;  in netlink_ring_setup_skb()
735 	struct netlink_ring *ring;  in netlink_mmap_sendmsg()  local
743 	ring   = &nlk->tx_ring;  in netlink_mmap_sendmsg()
744 	maxlen = ring->frame_size - NL_MMAP_HDRLEN;  in netlink_mmap_sendmsg()
749 		hdr = netlink_current_frame(ring, NL_MMAP_STATUS_VALID);  in netlink_mmap_sendmsg()
774 		netlink_increment_head(ring);  in netlink_mmap_sendmsg()
828 	struct netlink_ring *ring = &nlk->rx_ring;  in netlink_ring_set_copied()  local
832 	hdr = netlink_current_frame(ring, NL_MMAP_STATUS_UNUSED);  in netlink_ring_set_copied()
839 	netlink_increment_head(ring);  in netlink_ring_set_copied()
863 	struct netlink_ring *ring;  in netlink_skb_destructor()  local
877 			ring = &nlk_sk(sk)->tx_ring;  in netlink_skb_destructor()
883 			ring = &nlk_sk(sk)->rx_ring;  in netlink_skb_destructor()
886 		WARN_ON(atomic_read(&ring->pending) == 0);  in netlink_skb_destructor()
887 		atomic_dec(&ring->pending);  in netlink_skb_destructor()
1866 	struct netlink_ring *ring;  in netlink_alloc_skb()  local
1874 	ring = &nlk_sk(sk)->rx_ring;  in netlink_alloc_skb()
1876 	if (ring->pg_vec == NULL)  in netlink_alloc_skb()
1879 	if (ring->frame_size - NL_MMAP_HDRLEN < size)  in netlink_alloc_skb()
1888 	if (ring->pg_vec == NULL)  in netlink_alloc_skb()
1892 	maxlen = ring->frame_size - NL_MMAP_HDRLEN;  in netlink_alloc_skb()
1896 	netlink_forward_ring(ring);  in netlink_alloc_skb()
1897 	hdr = netlink_current_frame(ring, NL_MMAP_STATUS_UNUSED);  in netlink_alloc_skb()
1900 	netlink_ring_setup_skb(skb, sk, ring, hdr);  in netlink_alloc_skb()
1902 	atomic_inc(&ring->pending);  in netlink_alloc_skb()
1903 	netlink_increment_head(ring);  in netlink_alloc_skb()