382 			    struct tun_file *tfile)  in tun_flow_update()  argument
387 	u16 queue_index = tfile->queue_index;  in tun_flow_update()
398 	if (tun->numqueues == 1 || tfile->detached)  in tun_flow_update()
486 static void tun_disable_queue(struct tun_struct *tun, struct tun_file *tfile)  in tun_disable_queue()  argument
488 	tfile->detached = tun;  in tun_disable_queue()
489 	list_add_tail(&tfile->next, &tun->disabled);  in tun_disable_queue()
493 static struct tun_struct *tun_enable_queue(struct tun_file *tfile)  in tun_enable_queue()  argument
495 	struct tun_struct *tun = tfile->detached;  in tun_enable_queue()
497 	tfile->detached = NULL;  in tun_enable_queue()
498 	list_del_init(&tfile->next);  in tun_enable_queue()
503 static void tun_queue_purge(struct tun_file *tfile)  in tun_queue_purge()  argument
505 	skb_queue_purge(&tfile->sk.sk_receive_queue);  in tun_queue_purge()
506 	skb_queue_purge(&tfile->sk.sk_error_queue);  in tun_queue_purge()
509 static void __tun_detach(struct tun_file *tfile, bool clean)  in __tun_detach()  argument
514 	tun = rtnl_dereference(tfile->tun);  in __tun_detach()
516 	if (tun && !tfile->detached) {  in __tun_detach()
517 		u16 index = tfile->queue_index;  in __tun_detach()
527 			RCU_INIT_POINTER(tfile->tun, NULL);  in __tun_detach()
528 			sock_put(&tfile->sk);  in __tun_detach()
530 			tun_disable_queue(tun, tfile);  in __tun_detach()
535 		tun_queue_purge(tfile);  in __tun_detach()
537 	} else if (tfile->detached && clean) {  in __tun_detach()
538 		tun = tun_enable_queue(tfile);  in __tun_detach()
539 		sock_put(&tfile->sk);  in __tun_detach()
550 		sock_put(&tfile->sk);  in __tun_detach()
554 static void tun_detach(struct tun_file *tfile, bool clean)  in tun_detach()  argument
557 	__tun_detach(tfile, clean);  in tun_detach()
564 	struct tun_file *tfile, *tmp;  in tun_detach_all()  local
568 		tfile = rtnl_dereference(tun->tfiles[i]);  in tun_detach_all()
569 		BUG_ON(!tfile);  in tun_detach_all()
570 		tfile->socket.sk->sk_shutdown = RCV_SHUTDOWN;  in tun_detach_all()
571 		tfile->socket.sk->sk_data_ready(tfile->socket.sk);  in tun_detach_all()
572 		RCU_INIT_POINTER(tfile->tun, NULL);  in tun_detach_all()
575 	list_for_each_entry(tfile, &tun->disabled, next) {  in tun_detach_all()
576 		tfile->socket.sk->sk_shutdown = RCV_SHUTDOWN;  in tun_detach_all()
577 		tfile->socket.sk->sk_data_ready(tfile->socket.sk);  in tun_detach_all()
578 		RCU_INIT_POINTER(tfile->tun, NULL);  in tun_detach_all()
584 		tfile = rtnl_dereference(tun->tfiles[i]);  in tun_detach_all()
586 		tun_queue_purge(tfile);  in tun_detach_all()
587 		sock_put(&tfile->sk);  in tun_detach_all()
589 	list_for_each_entry_safe(tfile, tmp, &tun->disabled, next) {  in tun_detach_all()
590 		tun_enable_queue(tfile);  in tun_detach_all()
591 		tun_queue_purge(tfile);  in tun_detach_all()
592 		sock_put(&tfile->sk);  in tun_detach_all()
602 	struct tun_file *tfile = file->private_data;  in tun_attach()  local
605 	err = security_tun_dev_attach(tfile->socket.sk, tun->security);  in tun_attach()
610 	if (rtnl_dereference(tfile->tun) && !tfile->detached)  in tun_attach()
618 	if (!tfile->detached &&  in tun_attach()
626 		err = __sk_attach_filter(&tun->fprog, tfile->socket.sk,  in tun_attach()
631 	tfile->queue_index = tun->numqueues;  in tun_attach()
632 	tfile->socket.sk->sk_shutdown &= ~RCV_SHUTDOWN;  in tun_attach()
633 	rcu_assign_pointer(tfile->tun, tun);  in tun_attach()
634 	rcu_assign_pointer(tun->tfiles[tun->numqueues], tfile);  in tun_attach()
637 	if (tfile->detached)  in tun_attach()
638 		tun_enable_queue(tfile);  in tun_attach()
640 		sock_hold(&tfile->sk);  in tun_attach()
652 static struct tun_struct *__tun_get(struct tun_file *tfile)  in __tun_get()  argument
657 	tun = rcu_dereference(tfile->tun);  in __tun_get()
814 	struct tun_file *tfile;  in tun_net_xmit()  local
818 	tfile = rcu_dereference(tun->tfiles[txq]);  in tun_net_xmit()
843 	BUG_ON(!tfile);  in tun_net_xmit()
851 	if (tfile->socket.sk->sk_filter &&  in tun_net_xmit()
852 	    sk_filter(tfile->socket.sk, skb))  in tun_net_xmit()
858 	if (skb_queue_len(&tfile->socket.sk->sk_receive_queue) * numqueues  in tun_net_xmit()
878 	skb_queue_tail(&tfile->socket.sk->sk_receive_queue, skb);  in tun_net_xmit()
881 	if (tfile->flags & TUN_FASYNC)  in tun_net_xmit()
882 		kill_fasync(&tfile->fasync, SIGIO, POLL_IN);  in tun_net_xmit()
883 	tfile->socket.sk->sk_data_ready(tfile->socket.sk);  in tun_net_xmit()
1027 	struct tun_file *tfile = file->private_data;  in tun_chr_poll()  local
1028 	struct tun_struct *tun = __tun_get(tfile);  in tun_chr_poll()
1035 	sk = tfile->socket.sk;  in tun_chr_poll()
1058 static struct sk_buff *tun_alloc_skb(struct tun_file *tfile,  in tun_alloc_skb()  argument
1062 	struct sock *sk = tfile->socket.sk;  in tun_alloc_skb()
1084 static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile,  in tun_get_user()  argument
1161 	skb = tun_alloc_skb(tfile, align, copylen, linear, noblock);  in tun_get_user()
1268 	tun_flow_update(tun, rxhash, tfile);  in tun_get_user()
1276 	struct tun_file *tfile = file->private_data;  in tun_chr_write_iter()  local
1282 	result = tun_get_user(tun, tfile, NULL, from, file->f_flags & O_NONBLOCK);  in tun_chr_write_iter()
1290 			    struct tun_file *tfile,  in tun_put_user()  argument
1401 static ssize_t tun_do_read(struct tun_struct *tun, struct tun_file *tfile,  in tun_do_read()  argument
1415 	skb = __skb_recv_datagram(tfile->socket.sk, noblock ? MSG_DONTWAIT : 0,  in tun_do_read()
1420 	ret = tun_put_user(tun, tfile, skb, to);  in tun_do_read()
1432 	struct tun_file *tfile = file->private_data;  in tun_chr_read_iter()  local
1433 	struct tun_struct *tun = __tun_get(tfile);  in tun_chr_read_iter()
1438 	ret = tun_do_read(tun, tfile, to, file->f_flags & O_NONBLOCK);  in tun_chr_read_iter()
1486 	struct tun_file *tfile;  in tun_sock_write_space()  local
1500 	tfile = container_of(sk, struct tun_file, sk);  in tun_sock_write_space()
1501 	kill_fasync(&tfile->fasync, SIGIO, POLL_OUT);  in tun_sock_write_space()
1507 	struct tun_file *tfile = container_of(sock, struct tun_file, socket);  in tun_sendmsg()  local
1508 	struct tun_struct *tun = __tun_get(tfile);  in tun_sendmsg()
1513 	ret = tun_get_user(tun, tfile, m->msg_control, &m->msg_iter,  in tun_sendmsg()
1522 	struct tun_file *tfile = container_of(sock, struct tun_file, socket);  in tun_recvmsg()  local
1523 	struct tun_struct *tun = __tun_get(tfile);  in tun_recvmsg()
1538 	ret = tun_do_read(tun, tfile, &m->msg_iter, flags & MSG_DONTWAIT);  in tun_recvmsg()
1610 	struct tun_file *tfile = file->private_data;  in tun_set_iff()  local
1614 	if (tfile->detached)  in tun_set_iff()
1686 		dev->ifindex = tfile->ifindex;  in tun_set_iff()
1696 		tun->sndbuf = tfile->socket.sk->sk_sndbuf;  in tun_set_iff()
1804 	struct tun_file *tfile;  in tun_detach_filter()  local
1807 		tfile = rtnl_dereference(tun->tfiles[i]);  in tun_detach_filter()
1808 		__sk_detach_filter(tfile->socket.sk, lockdep_rtnl_is_held());  in tun_detach_filter()
1817 	struct tun_file *tfile;  in tun_attach_filter()  local
1820 		tfile = rtnl_dereference(tun->tfiles[i]);  in tun_attach_filter()
1821 		ret = __sk_attach_filter(&tun->fprog, tfile->socket.sk,  in tun_attach_filter()
1835 	struct tun_file *tfile;  in tun_set_sndbuf()  local
1839 		tfile = rtnl_dereference(tun->tfiles[i]);  in tun_set_sndbuf()
1840 		tfile->socket.sk->sk_sndbuf = tun->sndbuf;  in tun_set_sndbuf()
1846 	struct tun_file *tfile = file->private_data;  in tun_set_queue()  local
1853 		tun = tfile->detached;  in tun_set_queue()
1863 		tun = rtnl_dereference(tfile->tun);  in tun_set_queue()
1864 		if (!tun || !(tun->flags & IFF_MULTI_QUEUE) || tfile->detached)  in tun_set_queue()
1867 			__tun_detach(tfile, false);  in tun_set_queue()
1879 	struct tun_file *tfile = file->private_data;  in __tun_chr_ioctl()  local
1910 	tun = __tun_get(tfile);  in __tun_chr_ioctl()
1914 		ret = tun_set_iff(sock_net(&tfile->sk), file, &ifr);  in __tun_chr_ioctl()
1933 		tfile->ifindex = ifindex;  in __tun_chr_ioctl()
1948 		if (tfile->detached)  in __tun_chr_ioctl()
1950 		if (!tfile->socket.sk->sk_filter)  in __tun_chr_ioctl()
2054 		sndbuf = tfile->socket.sk->sk_sndbuf;  in __tun_chr_ioctl()
2193 	struct tun_file *tfile = file->private_data;  in tun_chr_fasync()  local
2196 	if ((ret = fasync_helper(fd, file, on, &tfile->fasync)) < 0)  in tun_chr_fasync()
2201 		tfile->flags |= TUN_FASYNC;  in tun_chr_fasync()
2203 		tfile->flags &= ~TUN_FASYNC;  in tun_chr_fasync()
2212 	struct tun_file *tfile;  in tun_chr_open()  local
2216 	tfile = (struct tun_file *)sk_alloc(net, AF_UNSPEC, GFP_KERNEL,  in tun_chr_open()
2218 	if (!tfile)  in tun_chr_open()
2220 	RCU_INIT_POINTER(tfile->tun, NULL);  in tun_chr_open()
2221 	tfile->flags = 0;  in tun_chr_open()
2222 	tfile->ifindex = 0;  in tun_chr_open()
2224 	init_waitqueue_head(&tfile->wq.wait);  in tun_chr_open()
2225 	RCU_INIT_POINTER(tfile->socket.wq, &tfile->wq);  in tun_chr_open()
2227 	tfile->socket.file = file;  in tun_chr_open()
2228 	tfile->socket.ops = &tun_socket_ops;  in tun_chr_open()
2230 	sock_init_data(&tfile->socket, &tfile->sk);  in tun_chr_open()
2232 	tfile->sk.sk_write_space = tun_sock_write_space;  in tun_chr_open()
2233 	tfile->sk.sk_sndbuf = INT_MAX;  in tun_chr_open()
2235 	file->private_data = tfile;  in tun_chr_open()
2236 	INIT_LIST_HEAD(&tfile->next);  in tun_chr_open()
2238 	sock_set_flag(&tfile->sk, SOCK_ZEROCOPY);  in tun_chr_open()
2245 	struct tun_file *tfile = file->private_data;  in tun_chr_close()  local
2247 	tun_detach(tfile, true);  in tun_chr_close()
2397 	struct tun_file *tfile;  in tun_get_socket()  local
2400 	tfile = file->private_data;  in tun_get_socket()
2401 	if (!tfile)  in tun_get_socket()
2403 	return &tfile->socket;  in tun_get_socket()