67 	delta = now - dev->power.accounting_timestamp;  in update_pm_runtime_accounting()
69 	dev->power.accounting_timestamp = now;  in update_pm_runtime_accounting()
71 	if (dev->power.disable_depth > 0)  in update_pm_runtime_accounting()
74 	if (dev->power.runtime_status == RPM_SUSPENDED)  in update_pm_runtime_accounting()
75 		dev->power.suspended_jiffies += delta;  in update_pm_runtime_accounting()
77 		dev->power.active_jiffies += delta;  in update_pm_runtime_accounting()
83 	dev->power.runtime_status = status;  in __update_runtime_status()
92 	if (dev->power.timer_expires > 0) {  in pm_runtime_deactivate_timer()
93 		del_timer(&dev->power.suspend_timer);  in pm_runtime_deactivate_timer()
94 		dev->power.timer_expires = 0;  in pm_runtime_deactivate_timer()
109 	dev->power.request = RPM_REQ_NONE;  in pm_runtime_cancel_pending()
131 	if (!dev->power.use_autosuspend)  in pm_runtime_autosuspend_expiration()
134 	autosuspend_delay = ACCESS_ONCE(dev->power.autosuspend_delay);  in pm_runtime_autosuspend_expiration()
138 	last_busy = ACCESS_ONCE(dev->power.last_busy);  in pm_runtime_autosuspend_expiration()
161 	return dev->power.memalloc_noio;  in dev_memalloc_noio()
201 		spin_lock_irq(&dev->power.lock);  in pm_runtime_set_memalloc_noio()
202 		enabled = dev->power.memalloc_noio;  in pm_runtime_set_memalloc_noio()
203 		dev->power.memalloc_noio = enable;  in pm_runtime_set_memalloc_noio()
204 		spin_unlock_irq(&dev->power.lock);  in pm_runtime_set_memalloc_noio()
237 	if (dev->power.runtime_error)  in rpm_check_suspend_allowed()
239 	else if (dev->power.disable_depth > 0)  in rpm_check_suspend_allowed()
241 	else if (atomic_read(&dev->power.usage_count) > 0)  in rpm_check_suspend_allowed()
247 	else if ((dev->power.deferred_resume  in rpm_check_suspend_allowed()
248 			&& dev->power.runtime_status == RPM_SUSPENDING)  in rpm_check_suspend_allowed()
249 	    || (dev->power.request_pending  in rpm_check_suspend_allowed()
250 			&& dev->power.request == RPM_REQ_RESUME))  in rpm_check_suspend_allowed()
254 	else if (dev->power.runtime_status == RPM_SUSPENDED)  in rpm_check_suspend_allowed()
266 	__releases(&dev->power.lock) __acquires(&dev->power.lock)  in __rpm_callback()
270 	if (dev->power.irq_safe)  in __rpm_callback()
271 		spin_unlock(&dev->power.lock);  in __rpm_callback()
273 		spin_unlock_irq(&dev->power.lock);  in __rpm_callback()
277 	if (dev->power.irq_safe)  in __rpm_callback()
278 		spin_lock(&dev->power.lock);  in __rpm_callback()
280 		spin_lock_irq(&dev->power.lock);  in __rpm_callback()
309 	else if (dev->power.runtime_status != RPM_ACTIVE)  in rpm_idle()
316 	else if (dev->power.request_pending &&  in rpm_idle()
317 	    dev->power.request > RPM_REQ_IDLE)  in rpm_idle()
321 	else if (dev->power.idle_notification)  in rpm_idle()
327 	dev->power.request = RPM_REQ_NONE;  in rpm_idle()
329 	if (dev->power.no_callbacks)  in rpm_idle()
334 		dev->power.request = RPM_REQ_IDLE;  in rpm_idle()
335 		if (!dev->power.request_pending) {  in rpm_idle()
336 			dev->power.request_pending = true;  in rpm_idle()
337 			queue_work(pm_wq, &dev->power.work);  in rpm_idle()
343 	dev->power.idle_notification = true;  in rpm_idle()
350 	dev->power.idle_notification = false;  in rpm_idle()
351 	wake_up_all(&dev->power.wait_queue);  in rpm_idle()
370 	if (dev->power.memalloc_noio) {  in rpm_callback()
389 	dev->power.runtime_error = retval;  in rpm_callback()
415 	__releases(&dev->power.lock) __acquires(&dev->power.lock)  in rpm_suspend()
430 	else if (dev->power.runtime_status == RPM_RESUMING &&  in rpm_suspend()
438 	    && dev->power.runtime_status != RPM_SUSPENDING) {  in rpm_suspend()
443 			dev->power.request = RPM_REQ_NONE;  in rpm_suspend()
452 			if (!(dev->power.timer_expires && time_before_eq(  in rpm_suspend()
453 			    dev->power.timer_expires, expires))) {  in rpm_suspend()
454 				dev->power.timer_expires = expires;  in rpm_suspend()
455 				mod_timer(&dev->power.suspend_timer, expires);  in rpm_suspend()
457 			dev->power.timer_autosuspends = 1;  in rpm_suspend()
465 	if (dev->power.runtime_status == RPM_SUSPENDING) {  in rpm_suspend()
473 		if (dev->power.irq_safe) {  in rpm_suspend()
474 			spin_unlock(&dev->power.lock);  in rpm_suspend()
478 			spin_lock(&dev->power.lock);  in rpm_suspend()
484 			prepare_to_wait(&dev->power.wait_queue, &wait,  in rpm_suspend()
486 			if (dev->power.runtime_status != RPM_SUSPENDING)  in rpm_suspend()
489 			spin_unlock_irq(&dev->power.lock);  in rpm_suspend()
493 			spin_lock_irq(&dev->power.lock);  in rpm_suspend()
495 		finish_wait(&dev->power.wait_queue, &wait);  in rpm_suspend()
499 	if (dev->power.no_callbacks)  in rpm_suspend()
504 		dev->power.request = (rpmflags & RPM_AUTO) ?  in rpm_suspend()
506 		if (!dev->power.request_pending) {  in rpm_suspend()
507 			dev->power.request_pending = true;  in rpm_suspend()
508 			queue_work(pm_wq, &dev->power.work);  in rpm_suspend()
527 		atomic_add_unless(&parent->power.child_count, -1, 0);  in rpm_suspend()
529 	wake_up_all(&dev->power.wait_queue);  in rpm_suspend()
531 	if (dev->power.deferred_resume) {  in rpm_suspend()
532 		dev->power.deferred_resume = false;  in rpm_suspend()
539 	if (parent && !parent->power.ignore_children && !dev->power.irq_safe) {  in rpm_suspend()
540 		spin_unlock(&dev->power.lock);  in rpm_suspend()
542 		spin_lock(&parent->power.lock);  in rpm_suspend()
544 		spin_unlock(&parent->power.lock);  in rpm_suspend()
546 		spin_lock(&dev->power.lock);  in rpm_suspend()
556 	dev->power.deferred_resume = false;  in rpm_suspend()
557 	wake_up_all(&dev->power.wait_queue);  in rpm_suspend()
560 		dev->power.runtime_error = 0;  in rpm_suspend()
595 	__releases(&dev->power.lock) __acquires(&dev->power.lock)  in rpm_resume()
604 	if (dev->power.runtime_error)  in rpm_resume()
606 	else if (dev->power.disable_depth == 1 && dev->power.is_suspended  in rpm_resume()
607 	    && dev->power.runtime_status == RPM_ACTIVE)  in rpm_resume()
609 	else if (dev->power.disable_depth > 0)  in rpm_resume()
620 	dev->power.request = RPM_REQ_NONE;  in rpm_resume()
621 	if (!dev->power.timer_autosuspends)  in rpm_resume()
624 	if (dev->power.runtime_status == RPM_ACTIVE) {  in rpm_resume()
629 	if (dev->power.runtime_status == RPM_RESUMING  in rpm_resume()
630 	    || dev->power.runtime_status == RPM_SUSPENDING) {  in rpm_resume()
634 			if (dev->power.runtime_status == RPM_SUSPENDING)  in rpm_resume()
635 				dev->power.deferred_resume = true;  in rpm_resume()
641 		if (dev->power.irq_safe) {  in rpm_resume()
642 			spin_unlock(&dev->power.lock);  in rpm_resume()
646 			spin_lock(&dev->power.lock);  in rpm_resume()
652 			prepare_to_wait(&dev->power.wait_queue, &wait,  in rpm_resume()
654 			if (dev->power.runtime_status != RPM_RESUMING  in rpm_resume()
655 			    && dev->power.runtime_status != RPM_SUSPENDING)  in rpm_resume()
658 			spin_unlock_irq(&dev->power.lock);  in rpm_resume()
662 			spin_lock_irq(&dev->power.lock);  in rpm_resume()
664 		finish_wait(&dev->power.wait_queue, &wait);  in rpm_resume()
673 	if (dev->power.no_callbacks && !parent && dev->parent) {  in rpm_resume()
674 		spin_lock_nested(&dev->parent->power.lock, SINGLE_DEPTH_NESTING);  in rpm_resume()
675 		if (dev->parent->power.disable_depth > 0  in rpm_resume()
676 		    || dev->parent->power.ignore_children  in rpm_resume()
677 		    || dev->parent->power.runtime_status == RPM_ACTIVE) {  in rpm_resume()
678 			atomic_inc(&dev->parent->power.child_count);  in rpm_resume()
679 			spin_unlock(&dev->parent->power.lock);  in rpm_resume()
683 		spin_unlock(&dev->parent->power.lock);  in rpm_resume()
688 		dev->power.request = RPM_REQ_RESUME;  in rpm_resume()
689 		if (!dev->power.request_pending) {  in rpm_resume()
690 			dev->power.request_pending = true;  in rpm_resume()
691 			queue_work(pm_wq, &dev->power.work);  in rpm_resume()
704 		if (dev->power.irq_safe)  in rpm_resume()
706 		spin_unlock(&dev->power.lock);  in rpm_resume()
710 		spin_lock(&parent->power.lock);  in rpm_resume()
715 		if (!parent->power.disable_depth  in rpm_resume()
716 		    && !parent->power.ignore_children) {  in rpm_resume()
718 			if (parent->power.runtime_status != RPM_ACTIVE)  in rpm_resume()
721 		spin_unlock(&parent->power.lock);  in rpm_resume()
723 		spin_lock(&dev->power.lock);  in rpm_resume()
730 	if (dev->power.no_callbacks)  in rpm_resume()
745 			atomic_inc(&parent->power.child_count);  in rpm_resume()
747 	wake_up_all(&dev->power.wait_queue);  in rpm_resume()
753 	if (parent && !dev->power.irq_safe) {  in rpm_resume()
754 		spin_unlock_irq(&dev->power.lock);  in rpm_resume()
758 		spin_lock_irq(&dev->power.lock);  in rpm_resume()
775 	struct device *dev = container_of(work, struct device, power.work);  in pm_runtime_work()
778 	spin_lock_irq(&dev->power.lock);  in pm_runtime_work()
780 	if (!dev->power.request_pending)  in pm_runtime_work()
783 	req = dev->power.request;  in pm_runtime_work()
784 	dev->power.request = RPM_REQ_NONE;  in pm_runtime_work()
785 	dev->power.request_pending = false;  in pm_runtime_work()
805 	spin_unlock_irq(&dev->power.lock);  in pm_runtime_work()
820 	spin_lock_irqsave(&dev->power.lock, flags);  in pm_suspend_timer_fn()
822 	expires = dev->power.timer_expires;  in pm_suspend_timer_fn()
825 		dev->power.timer_expires = 0;  in pm_suspend_timer_fn()
826 		rpm_suspend(dev, dev->power.timer_autosuspends ?  in pm_suspend_timer_fn()
830 	spin_unlock_irqrestore(&dev->power.lock, flags);  in pm_suspend_timer_fn()
843 	spin_lock_irqsave(&dev->power.lock, flags);  in pm_schedule_suspend()
857 	dev->power.timer_expires = jiffies + msecs_to_jiffies(delay);  in pm_schedule_suspend()
858 	dev->power.timer_expires += !dev->power.timer_expires;  in pm_schedule_suspend()
859 	dev->power.timer_autosuspends = 0;  in pm_schedule_suspend()
860 	mod_timer(&dev->power.suspend_timer, dev->power.timer_expires);  in pm_schedule_suspend()
863 	spin_unlock_irqrestore(&dev->power.lock, flags);  in pm_schedule_suspend()
886 	might_sleep_if(!(rpmflags & RPM_ASYNC) && !dev->power.irq_safe);  in __pm_runtime_idle()
889 		if (!atomic_dec_and_test(&dev->power.usage_count))  in __pm_runtime_idle()
893 	spin_lock_irqsave(&dev->power.lock, flags);  in __pm_runtime_idle()
895 	spin_unlock_irqrestore(&dev->power.lock, flags);  in __pm_runtime_idle()
918 	might_sleep_if(!(rpmflags & RPM_ASYNC) && !dev->power.irq_safe);  in __pm_runtime_suspend()
921 		if (!atomic_dec_and_test(&dev->power.usage_count))  in __pm_runtime_suspend()
925 	spin_lock_irqsave(&dev->power.lock, flags);  in __pm_runtime_suspend()
927 	spin_unlock_irqrestore(&dev->power.lock, flags);  in __pm_runtime_suspend()
949 	might_sleep_if(!(rpmflags & RPM_ASYNC) && !dev->power.irq_safe);  in __pm_runtime_resume()
952 		atomic_inc(&dev->power.usage_count);  in __pm_runtime_resume()
954 	spin_lock_irqsave(&dev->power.lock, flags);  in __pm_runtime_resume()
956 	spin_unlock_irqrestore(&dev->power.lock, flags);  in __pm_runtime_resume()
989 	spin_lock_irqsave(&dev->power.lock, flags);  in __pm_runtime_set_status()
991 	if (!dev->power.runtime_error && !dev->power.disable_depth) {  in __pm_runtime_set_status()
996 	if (dev->power.runtime_status == status)  in __pm_runtime_set_status()
1002 			atomic_add_unless(&parent->power.child_count, -1, 0);  in __pm_runtime_set_status()
1003 			notify_parent = !parent->power.ignore_children;  in __pm_runtime_set_status()
1009 		spin_lock_nested(&parent->power.lock, SINGLE_DEPTH_NESTING);  in __pm_runtime_set_status()
1016 		if (!parent->power.disable_depth  in __pm_runtime_set_status()
1017 		    && !parent->power.ignore_children  in __pm_runtime_set_status()
1018 		    && parent->power.runtime_status != RPM_ACTIVE)  in __pm_runtime_set_status()
1020 		else if (dev->power.runtime_status == RPM_SUSPENDED)  in __pm_runtime_set_status()
1021 			atomic_inc(&parent->power.child_count);  in __pm_runtime_set_status()
1023 		spin_unlock(&parent->power.lock);  in __pm_runtime_set_status()
1031 	dev->power.runtime_error = 0;  in __pm_runtime_set_status()
1033 	spin_unlock_irqrestore(&dev->power.lock, flags);  in __pm_runtime_set_status()
1055 	if (dev->power.request_pending) {  in __pm_runtime_barrier()
1056 		dev->power.request = RPM_REQ_NONE;  in __pm_runtime_barrier()
1057 		spin_unlock_irq(&dev->power.lock);  in __pm_runtime_barrier()
1059 		cancel_work_sync(&dev->power.work);  in __pm_runtime_barrier()
1061 		spin_lock_irq(&dev->power.lock);  in __pm_runtime_barrier()
1062 		dev->power.request_pending = false;  in __pm_runtime_barrier()
1065 	if (dev->power.runtime_status == RPM_SUSPENDING  in __pm_runtime_barrier()
1066 	    || dev->power.runtime_status == RPM_RESUMING  in __pm_runtime_barrier()
1067 	    || dev->power.idle_notification) {  in __pm_runtime_barrier()
1072 			prepare_to_wait(&dev->power.wait_queue, &wait,  in __pm_runtime_barrier()
1074 			if (dev->power.runtime_status != RPM_SUSPENDING  in __pm_runtime_barrier()
1075 			    && dev->power.runtime_status != RPM_RESUMING  in __pm_runtime_barrier()
1076 			    && !dev->power.idle_notification)  in __pm_runtime_barrier()
1078 			spin_unlock_irq(&dev->power.lock);  in __pm_runtime_barrier()
1082 			spin_lock_irq(&dev->power.lock);  in __pm_runtime_barrier()
1084 		finish_wait(&dev->power.wait_queue, &wait);  in __pm_runtime_barrier()
1107 	spin_lock_irq(&dev->power.lock);  in pm_runtime_barrier()
1109 	if (dev->power.request_pending  in pm_runtime_barrier()
1110 	    && dev->power.request == RPM_REQ_RESUME) {  in pm_runtime_barrier()
1117 	spin_unlock_irq(&dev->power.lock);  in pm_runtime_barrier()
1140 	spin_lock_irq(&dev->power.lock);  in __pm_runtime_disable()
1142 	if (dev->power.disable_depth > 0) {  in __pm_runtime_disable()
1143 		dev->power.disable_depth++;  in __pm_runtime_disable()
1152 	if (check_resume && dev->power.request_pending  in __pm_runtime_disable()
1153 	    && dev->power.request == RPM_REQ_RESUME) {  in __pm_runtime_disable()
1165 	if (!dev->power.disable_depth++)  in __pm_runtime_disable()
1169 	spin_unlock_irq(&dev->power.lock);  in __pm_runtime_disable()
1181 	spin_lock_irqsave(&dev->power.lock, flags);  in pm_runtime_enable()
1183 	if (dev->power.disable_depth > 0)  in pm_runtime_enable()
1184 		dev->power.disable_depth--;  in pm_runtime_enable()
1188 	spin_unlock_irqrestore(&dev->power.lock, flags);  in pm_runtime_enable()
1202 	spin_lock_irq(&dev->power.lock);  in pm_runtime_forbid()
1203 	if (!dev->power.runtime_auto)  in pm_runtime_forbid()
1206 	dev->power.runtime_auto = false;  in pm_runtime_forbid()
1207 	atomic_inc(&dev->power.usage_count);  in pm_runtime_forbid()
1211 	spin_unlock_irq(&dev->power.lock);  in pm_runtime_forbid()
1223 	spin_lock_irq(&dev->power.lock);  in pm_runtime_allow()
1224 	if (dev->power.runtime_auto)  in pm_runtime_allow()
1227 	dev->power.runtime_auto = true;  in pm_runtime_allow()
1228 	if (atomic_dec_and_test(&dev->power.usage_count))  in pm_runtime_allow()
1232 	spin_unlock_irq(&dev->power.lock);  in pm_runtime_allow()
1246 	spin_lock_irq(&dev->power.lock);  in pm_runtime_no_callbacks()
1247 	dev->power.no_callbacks = 1;  in pm_runtime_no_callbacks()
1248 	spin_unlock_irq(&dev->power.lock);  in pm_runtime_no_callbacks()
1269 	spin_lock_irq(&dev->power.lock);  in pm_runtime_irq_safe()
1270 	dev->power.irq_safe = 1;  in pm_runtime_irq_safe()
1271 	spin_unlock_irq(&dev->power.lock);  in pm_runtime_irq_safe()
1288 	int delay = dev->power.autosuspend_delay;  in update_autosuspend()
1291 	if (dev->power.use_autosuspend && delay < 0) {  in update_autosuspend()
1295 			atomic_inc(&dev->power.usage_count);  in update_autosuspend()
1305 			atomic_dec(&dev->power.usage_count);  in update_autosuspend()
1325 	spin_lock_irq(&dev->power.lock);  in pm_runtime_set_autosuspend_delay()
1326 	old_delay = dev->power.autosuspend_delay;  in pm_runtime_set_autosuspend_delay()
1327 	old_use = dev->power.use_autosuspend;  in pm_runtime_set_autosuspend_delay()
1328 	dev->power.autosuspend_delay = delay;  in pm_runtime_set_autosuspend_delay()
1330 	spin_unlock_irq(&dev->power.lock);  in pm_runtime_set_autosuspend_delay()
1346 	spin_lock_irq(&dev->power.lock);  in __pm_runtime_use_autosuspend()
1347 	old_delay = dev->power.autosuspend_delay;  in __pm_runtime_use_autosuspend()
1348 	old_use = dev->power.use_autosuspend;  in __pm_runtime_use_autosuspend()
1349 	dev->power.use_autosuspend = use;  in __pm_runtime_use_autosuspend()
1351 	spin_unlock_irq(&dev->power.lock);  in __pm_runtime_use_autosuspend()
1361 	dev->power.runtime_status = RPM_SUSPENDED;  in pm_runtime_init()
1362 	dev->power.idle_notification = false;  in pm_runtime_init()
1364 	dev->power.disable_depth = 1;  in pm_runtime_init()
1365 	atomic_set(&dev->power.usage_count, 0);  in pm_runtime_init()
1367 	dev->power.runtime_error = 0;  in pm_runtime_init()
1369 	atomic_set(&dev->power.child_count, 0);  in pm_runtime_init()
1371 	dev->power.runtime_auto = true;  in pm_runtime_init()
1373 	dev->power.request_pending = false;  in pm_runtime_init()
1374 	dev->power.request = RPM_REQ_NONE;  in pm_runtime_init()
1375 	dev->power.deferred_resume = false;  in pm_runtime_init()
1376 	dev->power.accounting_timestamp = jiffies;  in pm_runtime_init()
1377 	INIT_WORK(&dev->power.work, pm_runtime_work);  in pm_runtime_init()
1379 	dev->power.timer_expires = 0;  in pm_runtime_init()
1380 	setup_timer(&dev->power.suspend_timer, pm_suspend_timer_fn,  in pm_runtime_init()
1383 	init_waitqueue_head(&dev->power.wait_queue);  in pm_runtime_init()
1395 	if (dev->power.runtime_status == RPM_ACTIVE)  in pm_runtime_remove()
1397 	if (dev->power.irq_safe && dev->parent)  in pm_runtime_remove()