This source file includes following definitions.
- zero_zone_numa_counters
- zero_zones_numa_counters
- zero_global_numa_counters
- invalid_numa_statistics
- sysctl_vm_numa_stat_handler
- sum_vm_events
- all_vm_events
- vm_events_fold_cpu
- calculate_pressure_threshold
- calculate_normal_threshold
- refresh_zone_stat_thresholds
- set_pgdat_percpu_threshold
- __mod_zone_page_state
- __mod_node_page_state
- __inc_zone_state
- __inc_node_state
- __inc_zone_page_state
- __inc_node_page_state
- __dec_zone_state
- __dec_node_state
- __dec_zone_page_state
- __dec_node_page_state
- mod_zone_state
- mod_zone_page_state
- inc_zone_page_state
- dec_zone_page_state
- mod_node_state
- mod_node_page_state
- inc_node_state
- inc_node_page_state
- dec_node_page_state
- mod_zone_page_state
- inc_zone_page_state
- dec_zone_page_state
- inc_node_state
- mod_node_page_state
- inc_node_page_state
- dec_node_page_state
- fold_diff
- fold_diff
- refresh_cpu_vm_stats
- cpu_vm_stats_fold
- drain_zonestat
- __inc_numa_state
- sum_zone_node_page_state
- sum_zone_numa_state
- node_page_state
- fill_contig_page_info
- __fragmentation_index
- fragmentation_index
- frag_start
- frag_next
- frag_stop
- walk_zones_in_node
- frag_show_print
- frag_show
- pagetypeinfo_showfree_print
- pagetypeinfo_showfree
- pagetypeinfo_showblockcount_print
- pagetypeinfo_showblockcount
- pagetypeinfo_showmixedcount
- pagetypeinfo_show
- is_zone_first_populated
- zoneinfo_show_print
- zoneinfo_show
- vmstat_start
- vmstat_next
- vmstat_show
- vmstat_stop
- refresh_vm_stats
- vmstat_refresh
- vmstat_update
- need_update
- quiet_vmstat
- vmstat_shepherd
- start_shepherd_timer
- init_cpu_node_state
- vmstat_cpu_online
- vmstat_cpu_down_prep
- vmstat_cpu_dead
- init_mm_internals
- unusable_free_index
- unusable_show_print
- unusable_show
- unusable_open
- extfrag_show_print
- extfrag_show
- extfrag_open
- extfrag_debug_init
1
2
3
4
5
6
7
8
9
10
11
12
13 #include <linux/fs.h>
14 #include <linux/mm.h>
15 #include <linux/err.h>
16 #include <linux/module.h>
17 #include <linux/slab.h>
18 #include <linux/cpu.h>
19 #include <linux/cpumask.h>
20 #include <linux/vmstat.h>
21 #include <linux/proc_fs.h>
22 #include <linux/seq_file.h>
23 #include <linux/debugfs.h>
24 #include <linux/sched.h>
25 #include <linux/math64.h>
26 #include <linux/writeback.h>
27 #include <linux/compaction.h>
28 #include <linux/mm_inline.h>
29 #include <linux/page_ext.h>
30 #include <linux/page_owner.h>
31
32 #include "internal.h"
33
34 #define NUMA_STATS_THRESHOLD (U16_MAX - 2)
35
36 #ifdef CONFIG_NUMA
37 int sysctl_vm_numa_stat = ENABLE_NUMA_STAT;
38
39
40 static void zero_zone_numa_counters(struct zone *zone)
41 {
42 int item, cpu;
43
44 for (item = 0; item < NR_VM_NUMA_STAT_ITEMS; item++) {
45 atomic_long_set(&zone->vm_numa_stat[item], 0);
46 for_each_online_cpu(cpu)
47 per_cpu_ptr(zone->pageset, cpu)->vm_numa_stat_diff[item]
48 = 0;
49 }
50 }
51
52
53 static void zero_zones_numa_counters(void)
54 {
55 struct zone *zone;
56
57 for_each_populated_zone(zone)
58 zero_zone_numa_counters(zone);
59 }
60
61
62 static void zero_global_numa_counters(void)
63 {
64 int item;
65
66 for (item = 0; item < NR_VM_NUMA_STAT_ITEMS; item++)
67 atomic_long_set(&vm_numa_stat[item], 0);
68 }
69
70 static void invalid_numa_statistics(void)
71 {
72 zero_zones_numa_counters();
73 zero_global_numa_counters();
74 }
75
76 static DEFINE_MUTEX(vm_numa_stat_lock);
77
78 int sysctl_vm_numa_stat_handler(struct ctl_table *table, int write,
79 void __user *buffer, size_t *length, loff_t *ppos)
80 {
81 int ret, oldval;
82
83 mutex_lock(&vm_numa_stat_lock);
84 if (write)
85 oldval = sysctl_vm_numa_stat;
86 ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
87 if (ret || !write)
88 goto out;
89
90 if (oldval == sysctl_vm_numa_stat)
91 goto out;
92 else if (sysctl_vm_numa_stat == ENABLE_NUMA_STAT) {
93 static_branch_enable(&vm_numa_stat_key);
94 pr_info("enable numa statistics\n");
95 } else {
96 static_branch_disable(&vm_numa_stat_key);
97 invalid_numa_statistics();
98 pr_info("disable numa statistics, and clear numa counters\n");
99 }
100
101 out:
102 mutex_unlock(&vm_numa_stat_lock);
103 return ret;
104 }
105 #endif
106
107 #ifdef CONFIG_VM_EVENT_COUNTERS
108 DEFINE_PER_CPU(struct vm_event_state, vm_event_states) = {{0}};
109 EXPORT_PER_CPU_SYMBOL(vm_event_states);
110
111 static void sum_vm_events(unsigned long *ret)
112 {
113 int cpu;
114 int i;
115
116 memset(ret, 0, NR_VM_EVENT_ITEMS * sizeof(unsigned long));
117
118 for_each_online_cpu(cpu) {
119 struct vm_event_state *this = &per_cpu(vm_event_states, cpu);
120
121 for (i = 0; i < NR_VM_EVENT_ITEMS; i++)
122 ret[i] += this->event[i];
123 }
124 }
125
126
127
128
129
130
131 void all_vm_events(unsigned long *ret)
132 {
133 get_online_cpus();
134 sum_vm_events(ret);
135 put_online_cpus();
136 }
137 EXPORT_SYMBOL_GPL(all_vm_events);
138
139
140
141
142
143
144
145 void vm_events_fold_cpu(int cpu)
146 {
147 struct vm_event_state *fold_state = &per_cpu(vm_event_states, cpu);
148 int i;
149
150 for (i = 0; i < NR_VM_EVENT_ITEMS; i++) {
151 count_vm_events(i, fold_state->event[i]);
152 fold_state->event[i] = 0;
153 }
154 }
155
156 #endif
157
158
159
160
161
162
163 atomic_long_t vm_zone_stat[NR_VM_ZONE_STAT_ITEMS] __cacheline_aligned_in_smp;
164 atomic_long_t vm_numa_stat[NR_VM_NUMA_STAT_ITEMS] __cacheline_aligned_in_smp;
165 atomic_long_t vm_node_stat[NR_VM_NODE_STAT_ITEMS] __cacheline_aligned_in_smp;
166 EXPORT_SYMBOL(vm_zone_stat);
167 EXPORT_SYMBOL(vm_numa_stat);
168 EXPORT_SYMBOL(vm_node_stat);
169
170 #ifdef CONFIG_SMP
171
172 int calculate_pressure_threshold(struct zone *zone)
173 {
174 int threshold;
175 int watermark_distance;
176
177
178
179
180
181
182
183
184
185 watermark_distance = low_wmark_pages(zone) - min_wmark_pages(zone);
186 threshold = max(1, (int)(watermark_distance / num_online_cpus()));
187
188
189
190
191 threshold = min(125, threshold);
192
193 return threshold;
194 }
195
196 int calculate_normal_threshold(struct zone *zone)
197 {
198 int threshold;
199 int mem;
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231 mem = zone_managed_pages(zone) >> (27 - PAGE_SHIFT);
232
233 threshold = 2 * fls(num_online_cpus()) * (1 + fls(mem));
234
235
236
237
238 threshold = min(125, threshold);
239
240 return threshold;
241 }
242
243
244
245
246 void refresh_zone_stat_thresholds(void)
247 {
248 struct pglist_data *pgdat;
249 struct zone *zone;
250 int cpu;
251 int threshold;
252
253
254 for_each_online_pgdat(pgdat) {
255 for_each_online_cpu(cpu) {
256 per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->stat_threshold = 0;
257 }
258 }
259
260 for_each_populated_zone(zone) {
261 struct pglist_data *pgdat = zone->zone_pgdat;
262 unsigned long max_drift, tolerate_drift;
263
264 threshold = calculate_normal_threshold(zone);
265
266 for_each_online_cpu(cpu) {
267 int pgdat_threshold;
268
269 per_cpu_ptr(zone->pageset, cpu)->stat_threshold
270 = threshold;
271
272
273 pgdat_threshold = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->stat_threshold;
274 per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->stat_threshold
275 = max(threshold, pgdat_threshold);
276 }
277
278
279
280
281
282
283 tolerate_drift = low_wmark_pages(zone) - min_wmark_pages(zone);
284 max_drift = num_online_cpus() * threshold;
285 if (max_drift > tolerate_drift)
286 zone->percpu_drift_mark = high_wmark_pages(zone) +
287 max_drift;
288 }
289 }
290
291 void set_pgdat_percpu_threshold(pg_data_t *pgdat,
292 int (*calculate_pressure)(struct zone *))
293 {
294 struct zone *zone;
295 int cpu;
296 int threshold;
297 int i;
298
299 for (i = 0; i < pgdat->nr_zones; i++) {
300 zone = &pgdat->node_zones[i];
301 if (!zone->percpu_drift_mark)
302 continue;
303
304 threshold = (*calculate_pressure)(zone);
305 for_each_online_cpu(cpu)
306 per_cpu_ptr(zone->pageset, cpu)->stat_threshold
307 = threshold;
308 }
309 }
310
311
312
313
314
315
316 void __mod_zone_page_state(struct zone *zone, enum zone_stat_item item,
317 long delta)
318 {
319 struct per_cpu_pageset __percpu *pcp = zone->pageset;
320 s8 __percpu *p = pcp->vm_stat_diff + item;
321 long x;
322 long t;
323
324 x = delta + __this_cpu_read(*p);
325
326 t = __this_cpu_read(pcp->stat_threshold);
327
328 if (unlikely(x > t || x < -t)) {
329 zone_page_state_add(x, zone, item);
330 x = 0;
331 }
332 __this_cpu_write(*p, x);
333 }
334 EXPORT_SYMBOL(__mod_zone_page_state);
335
336 void __mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item,
337 long delta)
338 {
339 struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats;
340 s8 __percpu *p = pcp->vm_node_stat_diff + item;
341 long x;
342 long t;
343
344 x = delta + __this_cpu_read(*p);
345
346 t = __this_cpu_read(pcp->stat_threshold);
347
348 if (unlikely(x > t || x < -t)) {
349 node_page_state_add(x, pgdat, item);
350 x = 0;
351 }
352 __this_cpu_write(*p, x);
353 }
354 EXPORT_SYMBOL(__mod_node_page_state);
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379 void __inc_zone_state(struct zone *zone, enum zone_stat_item item)
380 {
381 struct per_cpu_pageset __percpu *pcp = zone->pageset;
382 s8 __percpu *p = pcp->vm_stat_diff + item;
383 s8 v, t;
384
385 v = __this_cpu_inc_return(*p);
386 t = __this_cpu_read(pcp->stat_threshold);
387 if (unlikely(v > t)) {
388 s8 overstep = t >> 1;
389
390 zone_page_state_add(v + overstep, zone, item);
391 __this_cpu_write(*p, -overstep);
392 }
393 }
394
395 void __inc_node_state(struct pglist_data *pgdat, enum node_stat_item item)
396 {
397 struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats;
398 s8 __percpu *p = pcp->vm_node_stat_diff + item;
399 s8 v, t;
400
401 v = __this_cpu_inc_return(*p);
402 t = __this_cpu_read(pcp->stat_threshold);
403 if (unlikely(v > t)) {
404 s8 overstep = t >> 1;
405
406 node_page_state_add(v + overstep, pgdat, item);
407 __this_cpu_write(*p, -overstep);
408 }
409 }
410
411 void __inc_zone_page_state(struct page *page, enum zone_stat_item item)
412 {
413 __inc_zone_state(page_zone(page), item);
414 }
415 EXPORT_SYMBOL(__inc_zone_page_state);
416
417 void __inc_node_page_state(struct page *page, enum node_stat_item item)
418 {
419 __inc_node_state(page_pgdat(page), item);
420 }
421 EXPORT_SYMBOL(__inc_node_page_state);
422
423 void __dec_zone_state(struct zone *zone, enum zone_stat_item item)
424 {
425 struct per_cpu_pageset __percpu *pcp = zone->pageset;
426 s8 __percpu *p = pcp->vm_stat_diff + item;
427 s8 v, t;
428
429 v = __this_cpu_dec_return(*p);
430 t = __this_cpu_read(pcp->stat_threshold);
431 if (unlikely(v < - t)) {
432 s8 overstep = t >> 1;
433
434 zone_page_state_add(v - overstep, zone, item);
435 __this_cpu_write(*p, overstep);
436 }
437 }
438
439 void __dec_node_state(struct pglist_data *pgdat, enum node_stat_item item)
440 {
441 struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats;
442 s8 __percpu *p = pcp->vm_node_stat_diff + item;
443 s8 v, t;
444
445 v = __this_cpu_dec_return(*p);
446 t = __this_cpu_read(pcp->stat_threshold);
447 if (unlikely(v < - t)) {
448 s8 overstep = t >> 1;
449
450 node_page_state_add(v - overstep, pgdat, item);
451 __this_cpu_write(*p, overstep);
452 }
453 }
454
455 void __dec_zone_page_state(struct page *page, enum zone_stat_item item)
456 {
457 __dec_zone_state(page_zone(page), item);
458 }
459 EXPORT_SYMBOL(__dec_zone_page_state);
460
461 void __dec_node_page_state(struct page *page, enum node_stat_item item)
462 {
463 __dec_node_state(page_pgdat(page), item);
464 }
465 EXPORT_SYMBOL(__dec_node_page_state);
466
467 #ifdef CONFIG_HAVE_CMPXCHG_LOCAL
468
469
470
471
472
473
474
475
476
477
478
479
480 static inline void mod_zone_state(struct zone *zone,
481 enum zone_stat_item item, long delta, int overstep_mode)
482 {
483 struct per_cpu_pageset __percpu *pcp = zone->pageset;
484 s8 __percpu *p = pcp->vm_stat_diff + item;
485 long o, n, t, z;
486
487 do {
488 z = 0;
489
490
491
492
493
494
495
496
497
498
499
500 t = this_cpu_read(pcp->stat_threshold);
501
502 o = this_cpu_read(*p);
503 n = delta + o;
504
505 if (n > t || n < -t) {
506 int os = overstep_mode * (t >> 1) ;
507
508
509 z = n + os;
510 n = -os;
511 }
512 } while (this_cpu_cmpxchg(*p, o, n) != o);
513
514 if (z)
515 zone_page_state_add(z, zone, item);
516 }
517
518 void mod_zone_page_state(struct zone *zone, enum zone_stat_item item,
519 long delta)
520 {
521 mod_zone_state(zone, item, delta, 0);
522 }
523 EXPORT_SYMBOL(mod_zone_page_state);
524
525 void inc_zone_page_state(struct page *page, enum zone_stat_item item)
526 {
527 mod_zone_state(page_zone(page), item, 1, 1);
528 }
529 EXPORT_SYMBOL(inc_zone_page_state);
530
531 void dec_zone_page_state(struct page *page, enum zone_stat_item item)
532 {
533 mod_zone_state(page_zone(page), item, -1, -1);
534 }
535 EXPORT_SYMBOL(dec_zone_page_state);
536
537 static inline void mod_node_state(struct pglist_data *pgdat,
538 enum node_stat_item item, int delta, int overstep_mode)
539 {
540 struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats;
541 s8 __percpu *p = pcp->vm_node_stat_diff + item;
542 long o, n, t, z;
543
544 do {
545 z = 0;
546
547
548
549
550
551
552
553
554
555
556
557 t = this_cpu_read(pcp->stat_threshold);
558
559 o = this_cpu_read(*p);
560 n = delta + o;
561
562 if (n > t || n < -t) {
563 int os = overstep_mode * (t >> 1) ;
564
565
566 z = n + os;
567 n = -os;
568 }
569 } while (this_cpu_cmpxchg(*p, o, n) != o);
570
571 if (z)
572 node_page_state_add(z, pgdat, item);
573 }
574
575 void mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item,
576 long delta)
577 {
578 mod_node_state(pgdat, item, delta, 0);
579 }
580 EXPORT_SYMBOL(mod_node_page_state);
581
582 void inc_node_state(struct pglist_data *pgdat, enum node_stat_item item)
583 {
584 mod_node_state(pgdat, item, 1, 1);
585 }
586
587 void inc_node_page_state(struct page *page, enum node_stat_item item)
588 {
589 mod_node_state(page_pgdat(page), item, 1, 1);
590 }
591 EXPORT_SYMBOL(inc_node_page_state);
592
593 void dec_node_page_state(struct page *page, enum node_stat_item item)
594 {
595 mod_node_state(page_pgdat(page), item, -1, -1);
596 }
597 EXPORT_SYMBOL(dec_node_page_state);
598 #else
599
600
601
602 void mod_zone_page_state(struct zone *zone, enum zone_stat_item item,
603 long delta)
604 {
605 unsigned long flags;
606
607 local_irq_save(flags);
608 __mod_zone_page_state(zone, item, delta);
609 local_irq_restore(flags);
610 }
611 EXPORT_SYMBOL(mod_zone_page_state);
612
613 void inc_zone_page_state(struct page *page, enum zone_stat_item item)
614 {
615 unsigned long flags;
616 struct zone *zone;
617
618 zone = page_zone(page);
619 local_irq_save(flags);
620 __inc_zone_state(zone, item);
621 local_irq_restore(flags);
622 }
623 EXPORT_SYMBOL(inc_zone_page_state);
624
625 void dec_zone_page_state(struct page *page, enum zone_stat_item item)
626 {
627 unsigned long flags;
628
629 local_irq_save(flags);
630 __dec_zone_page_state(page, item);
631 local_irq_restore(flags);
632 }
633 EXPORT_SYMBOL(dec_zone_page_state);
634
635 void inc_node_state(struct pglist_data *pgdat, enum node_stat_item item)
636 {
637 unsigned long flags;
638
639 local_irq_save(flags);
640 __inc_node_state(pgdat, item);
641 local_irq_restore(flags);
642 }
643 EXPORT_SYMBOL(inc_node_state);
644
645 void mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item,
646 long delta)
647 {
648 unsigned long flags;
649
650 local_irq_save(flags);
651 __mod_node_page_state(pgdat, item, delta);
652 local_irq_restore(flags);
653 }
654 EXPORT_SYMBOL(mod_node_page_state);
655
656 void inc_node_page_state(struct page *page, enum node_stat_item item)
657 {
658 unsigned long flags;
659 struct pglist_data *pgdat;
660
661 pgdat = page_pgdat(page);
662 local_irq_save(flags);
663 __inc_node_state(pgdat, item);
664 local_irq_restore(flags);
665 }
666 EXPORT_SYMBOL(inc_node_page_state);
667
668 void dec_node_page_state(struct page *page, enum node_stat_item item)
669 {
670 unsigned long flags;
671
672 local_irq_save(flags);
673 __dec_node_page_state(page, item);
674 local_irq_restore(flags);
675 }
676 EXPORT_SYMBOL(dec_node_page_state);
677 #endif
678
679
680
681
682
683 #ifdef CONFIG_NUMA
684 static int fold_diff(int *zone_diff, int *numa_diff, int *node_diff)
685 {
686 int i;
687 int changes = 0;
688
689 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
690 if (zone_diff[i]) {
691 atomic_long_add(zone_diff[i], &vm_zone_stat[i]);
692 changes++;
693 }
694
695 for (i = 0; i < NR_VM_NUMA_STAT_ITEMS; i++)
696 if (numa_diff[i]) {
697 atomic_long_add(numa_diff[i], &vm_numa_stat[i]);
698 changes++;
699 }
700
701 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
702 if (node_diff[i]) {
703 atomic_long_add(node_diff[i], &vm_node_stat[i]);
704 changes++;
705 }
706 return changes;
707 }
708 #else
709 static int fold_diff(int *zone_diff, int *node_diff)
710 {
711 int i;
712 int changes = 0;
713
714 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
715 if (zone_diff[i]) {
716 atomic_long_add(zone_diff[i], &vm_zone_stat[i]);
717 changes++;
718 }
719
720 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
721 if (node_diff[i]) {
722 atomic_long_add(node_diff[i], &vm_node_stat[i]);
723 changes++;
724 }
725 return changes;
726 }
727 #endif
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745 static int refresh_cpu_vm_stats(bool do_pagesets)
746 {
747 struct pglist_data *pgdat;
748 struct zone *zone;
749 int i;
750 int global_zone_diff[NR_VM_ZONE_STAT_ITEMS] = { 0, };
751 #ifdef CONFIG_NUMA
752 int global_numa_diff[NR_VM_NUMA_STAT_ITEMS] = { 0, };
753 #endif
754 int global_node_diff[NR_VM_NODE_STAT_ITEMS] = { 0, };
755 int changes = 0;
756
757 for_each_populated_zone(zone) {
758 struct per_cpu_pageset __percpu *p = zone->pageset;
759
760 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) {
761 int v;
762
763 v = this_cpu_xchg(p->vm_stat_diff[i], 0);
764 if (v) {
765
766 atomic_long_add(v, &zone->vm_stat[i]);
767 global_zone_diff[i] += v;
768 #ifdef CONFIG_NUMA
769
770 __this_cpu_write(p->expire, 3);
771 #endif
772 }
773 }
774 #ifdef CONFIG_NUMA
775 for (i = 0; i < NR_VM_NUMA_STAT_ITEMS; i++) {
776 int v;
777
778 v = this_cpu_xchg(p->vm_numa_stat_diff[i], 0);
779 if (v) {
780
781 atomic_long_add(v, &zone->vm_numa_stat[i]);
782 global_numa_diff[i] += v;
783 __this_cpu_write(p->expire, 3);
784 }
785 }
786
787 if (do_pagesets) {
788 cond_resched();
789
790
791
792
793
794
795
796 if (!__this_cpu_read(p->expire) ||
797 !__this_cpu_read(p->pcp.count))
798 continue;
799
800
801
802
803 if (zone_to_nid(zone) == numa_node_id()) {
804 __this_cpu_write(p->expire, 0);
805 continue;
806 }
807
808 if (__this_cpu_dec_return(p->expire))
809 continue;
810
811 if (__this_cpu_read(p->pcp.count)) {
812 drain_zone_pages(zone, this_cpu_ptr(&p->pcp));
813 changes++;
814 }
815 }
816 #endif
817 }
818
819 for_each_online_pgdat(pgdat) {
820 struct per_cpu_nodestat __percpu *p = pgdat->per_cpu_nodestats;
821
822 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) {
823 int v;
824
825 v = this_cpu_xchg(p->vm_node_stat_diff[i], 0);
826 if (v) {
827 atomic_long_add(v, &pgdat->vm_stat[i]);
828 global_node_diff[i] += v;
829 }
830 }
831 }
832
833 #ifdef CONFIG_NUMA
834 changes += fold_diff(global_zone_diff, global_numa_diff,
835 global_node_diff);
836 #else
837 changes += fold_diff(global_zone_diff, global_node_diff);
838 #endif
839 return changes;
840 }
841
842
843
844
845
846
847 void cpu_vm_stats_fold(int cpu)
848 {
849 struct pglist_data *pgdat;
850 struct zone *zone;
851 int i;
852 int global_zone_diff[NR_VM_ZONE_STAT_ITEMS] = { 0, };
853 #ifdef CONFIG_NUMA
854 int global_numa_diff[NR_VM_NUMA_STAT_ITEMS] = { 0, };
855 #endif
856 int global_node_diff[NR_VM_NODE_STAT_ITEMS] = { 0, };
857
858 for_each_populated_zone(zone) {
859 struct per_cpu_pageset *p;
860
861 p = per_cpu_ptr(zone->pageset, cpu);
862
863 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
864 if (p->vm_stat_diff[i]) {
865 int v;
866
867 v = p->vm_stat_diff[i];
868 p->vm_stat_diff[i] = 0;
869 atomic_long_add(v, &zone->vm_stat[i]);
870 global_zone_diff[i] += v;
871 }
872
873 #ifdef CONFIG_NUMA
874 for (i = 0; i < NR_VM_NUMA_STAT_ITEMS; i++)
875 if (p->vm_numa_stat_diff[i]) {
876 int v;
877
878 v = p->vm_numa_stat_diff[i];
879 p->vm_numa_stat_diff[i] = 0;
880 atomic_long_add(v, &zone->vm_numa_stat[i]);
881 global_numa_diff[i] += v;
882 }
883 #endif
884 }
885
886 for_each_online_pgdat(pgdat) {
887 struct per_cpu_nodestat *p;
888
889 p = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu);
890
891 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
892 if (p->vm_node_stat_diff[i]) {
893 int v;
894
895 v = p->vm_node_stat_diff[i];
896 p->vm_node_stat_diff[i] = 0;
897 atomic_long_add(v, &pgdat->vm_stat[i]);
898 global_node_diff[i] += v;
899 }
900 }
901
902 #ifdef CONFIG_NUMA
903 fold_diff(global_zone_diff, global_numa_diff, global_node_diff);
904 #else
905 fold_diff(global_zone_diff, global_node_diff);
906 #endif
907 }
908
909
910
911
912
913 void drain_zonestat(struct zone *zone, struct per_cpu_pageset *pset)
914 {
915 int i;
916
917 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
918 if (pset->vm_stat_diff[i]) {
919 int v = pset->vm_stat_diff[i];
920 pset->vm_stat_diff[i] = 0;
921 atomic_long_add(v, &zone->vm_stat[i]);
922 atomic_long_add(v, &vm_zone_stat[i]);
923 }
924
925 #ifdef CONFIG_NUMA
926 for (i = 0; i < NR_VM_NUMA_STAT_ITEMS; i++)
927 if (pset->vm_numa_stat_diff[i]) {
928 int v = pset->vm_numa_stat_diff[i];
929
930 pset->vm_numa_stat_diff[i] = 0;
931 atomic_long_add(v, &zone->vm_numa_stat[i]);
932 atomic_long_add(v, &vm_numa_stat[i]);
933 }
934 #endif
935 }
936 #endif
937
938 #ifdef CONFIG_NUMA
939 void __inc_numa_state(struct zone *zone,
940 enum numa_stat_item item)
941 {
942 struct per_cpu_pageset __percpu *pcp = zone->pageset;
943 u16 __percpu *p = pcp->vm_numa_stat_diff + item;
944 u16 v;
945
946 v = __this_cpu_inc_return(*p);
947
948 if (unlikely(v > NUMA_STATS_THRESHOLD)) {
949 zone_numa_state_add(v, zone, item);
950 __this_cpu_write(*p, 0);
951 }
952 }
953
954
955
956
957
958
959 unsigned long sum_zone_node_page_state(int node,
960 enum zone_stat_item item)
961 {
962 struct zone *zones = NODE_DATA(node)->node_zones;
963 int i;
964 unsigned long count = 0;
965
966 for (i = 0; i < MAX_NR_ZONES; i++)
967 count += zone_page_state(zones + i, item);
968
969 return count;
970 }
971
972
973
974
975
976 unsigned long sum_zone_numa_state(int node,
977 enum numa_stat_item item)
978 {
979 struct zone *zones = NODE_DATA(node)->node_zones;
980 int i;
981 unsigned long count = 0;
982
983 for (i = 0; i < MAX_NR_ZONES; i++)
984 count += zone_numa_state_snapshot(zones + i, item);
985
986 return count;
987 }
988
989
990
991
992 unsigned long node_page_state(struct pglist_data *pgdat,
993 enum node_stat_item item)
994 {
995 long x = atomic_long_read(&pgdat->vm_stat[item]);
996 #ifdef CONFIG_SMP
997 if (x < 0)
998 x = 0;
999 #endif
1000 return x;
1001 }
1002 #endif
1003
1004 #ifdef CONFIG_COMPACTION
1005
1006 struct contig_page_info {
1007 unsigned long free_pages;
1008 unsigned long free_blocks_total;
1009 unsigned long free_blocks_suitable;
1010 };
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020 static void fill_contig_page_info(struct zone *zone,
1021 unsigned int suitable_order,
1022 struct contig_page_info *info)
1023 {
1024 unsigned int order;
1025
1026 info->free_pages = 0;
1027 info->free_blocks_total = 0;
1028 info->free_blocks_suitable = 0;
1029
1030 for (order = 0; order < MAX_ORDER; order++) {
1031 unsigned long blocks;
1032
1033
1034 blocks = zone->free_area[order].nr_free;
1035 info->free_blocks_total += blocks;
1036
1037
1038 info->free_pages += blocks << order;
1039
1040
1041 if (order >= suitable_order)
1042 info->free_blocks_suitable += blocks <<
1043 (order - suitable_order);
1044 }
1045 }
1046
1047
1048
1049
1050
1051
1052
1053
1054 static int __fragmentation_index(unsigned int order, struct contig_page_info *info)
1055 {
1056 unsigned long requested = 1UL << order;
1057
1058 if (WARN_ON_ONCE(order >= MAX_ORDER))
1059 return 0;
1060
1061 if (!info->free_blocks_total)
1062 return 0;
1063
1064
1065 if (info->free_blocks_suitable)
1066 return -1000;
1067
1068
1069
1070
1071
1072
1073
1074 return 1000 - div_u64( (1000+(div_u64(info->free_pages * 1000ULL, requested))), info->free_blocks_total);
1075 }
1076
1077
1078 int fragmentation_index(struct zone *zone, unsigned int order)
1079 {
1080 struct contig_page_info info;
1081
1082 fill_contig_page_info(zone, order, &info);
1083 return __fragmentation_index(order, &info);
1084 }
1085 #endif
1086
1087 #if defined(CONFIG_PROC_FS) || defined(CONFIG_SYSFS) || defined(CONFIG_NUMA)
1088 #ifdef CONFIG_ZONE_DMA
1089 #define TEXT_FOR_DMA(xx) xx "_dma",
1090 #else
1091 #define TEXT_FOR_DMA(xx)
1092 #endif
1093
1094 #ifdef CONFIG_ZONE_DMA32
1095 #define TEXT_FOR_DMA32(xx) xx "_dma32",
1096 #else
1097 #define TEXT_FOR_DMA32(xx)
1098 #endif
1099
1100 #ifdef CONFIG_HIGHMEM
1101 #define TEXT_FOR_HIGHMEM(xx) xx "_high",
1102 #else
1103 #define TEXT_FOR_HIGHMEM(xx)
1104 #endif
1105
1106 #define TEXTS_FOR_ZONES(xx) TEXT_FOR_DMA(xx) TEXT_FOR_DMA32(xx) xx "_normal", \
1107 TEXT_FOR_HIGHMEM(xx) xx "_movable",
1108
1109 const char * const vmstat_text[] = {
1110
1111 "nr_free_pages",
1112 "nr_zone_inactive_anon",
1113 "nr_zone_active_anon",
1114 "nr_zone_inactive_file",
1115 "nr_zone_active_file",
1116 "nr_zone_unevictable",
1117 "nr_zone_write_pending",
1118 "nr_mlock",
1119 "nr_page_table_pages",
1120 "nr_kernel_stack",
1121 "nr_bounce",
1122 #if IS_ENABLED(CONFIG_ZSMALLOC)
1123 "nr_zspages",
1124 #endif
1125 "nr_free_cma",
1126
1127
1128 #ifdef CONFIG_NUMA
1129 "numa_hit",
1130 "numa_miss",
1131 "numa_foreign",
1132 "numa_interleave",
1133 "numa_local",
1134 "numa_other",
1135 #endif
1136
1137
1138 "nr_inactive_anon",
1139 "nr_active_anon",
1140 "nr_inactive_file",
1141 "nr_active_file",
1142 "nr_unevictable",
1143 "nr_slab_reclaimable",
1144 "nr_slab_unreclaimable",
1145 "nr_isolated_anon",
1146 "nr_isolated_file",
1147 "workingset_nodes",
1148 "workingset_refault",
1149 "workingset_activate",
1150 "workingset_restore",
1151 "workingset_nodereclaim",
1152 "nr_anon_pages",
1153 "nr_mapped",
1154 "nr_file_pages",
1155 "nr_dirty",
1156 "nr_writeback",
1157 "nr_writeback_temp",
1158 "nr_shmem",
1159 "nr_shmem_hugepages",
1160 "nr_shmem_pmdmapped",
1161 "nr_file_hugepages",
1162 "nr_file_pmdmapped",
1163 "nr_anon_transparent_hugepages",
1164 "nr_unstable",
1165 "nr_vmscan_write",
1166 "nr_vmscan_immediate_reclaim",
1167 "nr_dirtied",
1168 "nr_written",
1169 "nr_kernel_misc_reclaimable",
1170
1171
1172 "nr_dirty_threshold",
1173 "nr_dirty_background_threshold",
1174
1175 #ifdef CONFIG_VM_EVENT_COUNTERS
1176
1177 "pgpgin",
1178 "pgpgout",
1179 "pswpin",
1180 "pswpout",
1181
1182 TEXTS_FOR_ZONES("pgalloc")
1183 TEXTS_FOR_ZONES("allocstall")
1184 TEXTS_FOR_ZONES("pgskip")
1185
1186 "pgfree",
1187 "pgactivate",
1188 "pgdeactivate",
1189 "pglazyfree",
1190
1191 "pgfault",
1192 "pgmajfault",
1193 "pglazyfreed",
1194
1195 "pgrefill",
1196 "pgsteal_kswapd",
1197 "pgsteal_direct",
1198 "pgscan_kswapd",
1199 "pgscan_direct",
1200 "pgscan_direct_throttle",
1201
1202 #ifdef CONFIG_NUMA
1203 "zone_reclaim_failed",
1204 #endif
1205 "pginodesteal",
1206 "slabs_scanned",
1207 "kswapd_inodesteal",
1208 "kswapd_low_wmark_hit_quickly",
1209 "kswapd_high_wmark_hit_quickly",
1210 "pageoutrun",
1211
1212 "pgrotated",
1213
1214 "drop_pagecache",
1215 "drop_slab",
1216 "oom_kill",
1217
1218 #ifdef CONFIG_NUMA_BALANCING
1219 "numa_pte_updates",
1220 "numa_huge_pte_updates",
1221 "numa_hint_faults",
1222 "numa_hint_faults_local",
1223 "numa_pages_migrated",
1224 #endif
1225 #ifdef CONFIG_MIGRATION
1226 "pgmigrate_success",
1227 "pgmigrate_fail",
1228 #endif
1229 #ifdef CONFIG_COMPACTION
1230 "compact_migrate_scanned",
1231 "compact_free_scanned",
1232 "compact_isolated",
1233 "compact_stall",
1234 "compact_fail",
1235 "compact_success",
1236 "compact_daemon_wake",
1237 "compact_daemon_migrate_scanned",
1238 "compact_daemon_free_scanned",
1239 #endif
1240
1241 #ifdef CONFIG_HUGETLB_PAGE
1242 "htlb_buddy_alloc_success",
1243 "htlb_buddy_alloc_fail",
1244 #endif
1245 "unevictable_pgs_culled",
1246 "unevictable_pgs_scanned",
1247 "unevictable_pgs_rescued",
1248 "unevictable_pgs_mlocked",
1249 "unevictable_pgs_munlocked",
1250 "unevictable_pgs_cleared",
1251 "unevictable_pgs_stranded",
1252
1253 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1254 "thp_fault_alloc",
1255 "thp_fault_fallback",
1256 "thp_collapse_alloc",
1257 "thp_collapse_alloc_failed",
1258 "thp_file_alloc",
1259 "thp_file_mapped",
1260 "thp_split_page",
1261 "thp_split_page_failed",
1262 "thp_deferred_split_page",
1263 "thp_split_pmd",
1264 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
1265 "thp_split_pud",
1266 #endif
1267 "thp_zero_page_alloc",
1268 "thp_zero_page_alloc_failed",
1269 "thp_swpout",
1270 "thp_swpout_fallback",
1271 #endif
1272 #ifdef CONFIG_MEMORY_BALLOON
1273 "balloon_inflate",
1274 "balloon_deflate",
1275 #ifdef CONFIG_BALLOON_COMPACTION
1276 "balloon_migrate",
1277 #endif
1278 #endif
1279 #ifdef CONFIG_DEBUG_TLBFLUSH
1280 "nr_tlb_remote_flush",
1281 "nr_tlb_remote_flush_received",
1282 "nr_tlb_local_flush_all",
1283 "nr_tlb_local_flush_one",
1284 #endif
1285
1286 #ifdef CONFIG_DEBUG_VM_VMACACHE
1287 "vmacache_find_calls",
1288 "vmacache_find_hits",
1289 #endif
1290 #ifdef CONFIG_SWAP
1291 "swap_ra",
1292 "swap_ra_hit",
1293 #endif
1294 #endif
1295 };
1296 #endif
1297
1298 #if (defined(CONFIG_DEBUG_FS) && defined(CONFIG_COMPACTION)) || \
1299 defined(CONFIG_PROC_FS)
1300 static void *frag_start(struct seq_file *m, loff_t *pos)
1301 {
1302 pg_data_t *pgdat;
1303 loff_t node = *pos;
1304
1305 for (pgdat = first_online_pgdat();
1306 pgdat && node;
1307 pgdat = next_online_pgdat(pgdat))
1308 --node;
1309
1310 return pgdat;
1311 }
1312
1313 static void *frag_next(struct seq_file *m, void *arg, loff_t *pos)
1314 {
1315 pg_data_t *pgdat = (pg_data_t *)arg;
1316
1317 (*pos)++;
1318 return next_online_pgdat(pgdat);
1319 }
1320
1321 static void frag_stop(struct seq_file *m, void *arg)
1322 {
1323 }
1324
1325
1326
1327
1328
1329 static void walk_zones_in_node(struct seq_file *m, pg_data_t *pgdat,
1330 bool assert_populated, bool nolock,
1331 void (*print)(struct seq_file *m, pg_data_t *, struct zone *))
1332 {
1333 struct zone *zone;
1334 struct zone *node_zones = pgdat->node_zones;
1335 unsigned long flags;
1336
1337 for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) {
1338 if (assert_populated && !populated_zone(zone))
1339 continue;
1340
1341 if (!nolock)
1342 spin_lock_irqsave(&zone->lock, flags);
1343 print(m, pgdat, zone);
1344 if (!nolock)
1345 spin_unlock_irqrestore(&zone->lock, flags);
1346 }
1347 }
1348 #endif
1349
1350 #ifdef CONFIG_PROC_FS
1351 static void frag_show_print(struct seq_file *m, pg_data_t *pgdat,
1352 struct zone *zone)
1353 {
1354 int order;
1355
1356 seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name);
1357 for (order = 0; order < MAX_ORDER; ++order)
1358 seq_printf(m, "%6lu ", zone->free_area[order].nr_free);
1359 seq_putc(m, '\n');
1360 }
1361
1362
1363
1364
1365 static int frag_show(struct seq_file *m, void *arg)
1366 {
1367 pg_data_t *pgdat = (pg_data_t *)arg;
1368 walk_zones_in_node(m, pgdat, true, false, frag_show_print);
1369 return 0;
1370 }
1371
1372 static void pagetypeinfo_showfree_print(struct seq_file *m,
1373 pg_data_t *pgdat, struct zone *zone)
1374 {
1375 int order, mtype;
1376
1377 for (mtype = 0; mtype < MIGRATE_TYPES; mtype++) {
1378 seq_printf(m, "Node %4d, zone %8s, type %12s ",
1379 pgdat->node_id,
1380 zone->name,
1381 migratetype_names[mtype]);
1382 for (order = 0; order < MAX_ORDER; ++order) {
1383 unsigned long freecount = 0;
1384 struct free_area *area;
1385 struct list_head *curr;
1386 bool overflow = false;
1387
1388 area = &(zone->free_area[order]);
1389
1390 list_for_each(curr, &area->free_list[mtype]) {
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400 if (++freecount >= 100000) {
1401 overflow = true;
1402 break;
1403 }
1404 }
1405 seq_printf(m, "%s%6lu ", overflow ? ">" : "", freecount);
1406 spin_unlock_irq(&zone->lock);
1407 cond_resched();
1408 spin_lock_irq(&zone->lock);
1409 }
1410 seq_putc(m, '\n');
1411 }
1412 }
1413
1414
1415 static int pagetypeinfo_showfree(struct seq_file *m, void *arg)
1416 {
1417 int order;
1418 pg_data_t *pgdat = (pg_data_t *)arg;
1419
1420
1421 seq_printf(m, "%-43s ", "Free pages count per migrate type at order");
1422 for (order = 0; order < MAX_ORDER; ++order)
1423 seq_printf(m, "%6d ", order);
1424 seq_putc(m, '\n');
1425
1426 walk_zones_in_node(m, pgdat, true, false, pagetypeinfo_showfree_print);
1427
1428 return 0;
1429 }
1430
1431 static void pagetypeinfo_showblockcount_print(struct seq_file *m,
1432 pg_data_t *pgdat, struct zone *zone)
1433 {
1434 int mtype;
1435 unsigned long pfn;
1436 unsigned long start_pfn = zone->zone_start_pfn;
1437 unsigned long end_pfn = zone_end_pfn(zone);
1438 unsigned long count[MIGRATE_TYPES] = { 0, };
1439
1440 for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) {
1441 struct page *page;
1442
1443 page = pfn_to_online_page(pfn);
1444 if (!page)
1445 continue;
1446
1447
1448 if (!memmap_valid_within(pfn, page, zone))
1449 continue;
1450
1451 if (page_zone(page) != zone)
1452 continue;
1453
1454 mtype = get_pageblock_migratetype(page);
1455
1456 if (mtype < MIGRATE_TYPES)
1457 count[mtype]++;
1458 }
1459
1460
1461 seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name);
1462 for (mtype = 0; mtype < MIGRATE_TYPES; mtype++)
1463 seq_printf(m, "%12lu ", count[mtype]);
1464 seq_putc(m, '\n');
1465 }
1466
1467
1468 static int pagetypeinfo_showblockcount(struct seq_file *m, void *arg)
1469 {
1470 int mtype;
1471 pg_data_t *pgdat = (pg_data_t *)arg;
1472
1473 seq_printf(m, "\n%-23s", "Number of blocks type ");
1474 for (mtype = 0; mtype < MIGRATE_TYPES; mtype++)
1475 seq_printf(m, "%12s ", migratetype_names[mtype]);
1476 seq_putc(m, '\n');
1477 walk_zones_in_node(m, pgdat, true, false,
1478 pagetypeinfo_showblockcount_print);
1479
1480 return 0;
1481 }
1482
1483
1484
1485
1486
1487
1488
1489 static void pagetypeinfo_showmixedcount(struct seq_file *m, pg_data_t *pgdat)
1490 {
1491 #ifdef CONFIG_PAGE_OWNER
1492 int mtype;
1493
1494 if (!static_branch_unlikely(&page_owner_inited))
1495 return;
1496
1497 drain_all_pages(NULL);
1498
1499 seq_printf(m, "\n%-23s", "Number of mixed blocks ");
1500 for (mtype = 0; mtype < MIGRATE_TYPES; mtype++)
1501 seq_printf(m, "%12s ", migratetype_names[mtype]);
1502 seq_putc(m, '\n');
1503
1504 walk_zones_in_node(m, pgdat, true, true,
1505 pagetypeinfo_showmixedcount_print);
1506 #endif
1507 }
1508
1509
1510
1511
1512
1513 static int pagetypeinfo_show(struct seq_file *m, void *arg)
1514 {
1515 pg_data_t *pgdat = (pg_data_t *)arg;
1516
1517
1518 if (!node_state(pgdat->node_id, N_MEMORY))
1519 return 0;
1520
1521 seq_printf(m, "Page block order: %d\n", pageblock_order);
1522 seq_printf(m, "Pages per block: %lu\n", pageblock_nr_pages);
1523 seq_putc(m, '\n');
1524 pagetypeinfo_showfree(m, pgdat);
1525 pagetypeinfo_showblockcount(m, pgdat);
1526 pagetypeinfo_showmixedcount(m, pgdat);
1527
1528 return 0;
1529 }
1530
1531 static const struct seq_operations fragmentation_op = {
1532 .start = frag_start,
1533 .next = frag_next,
1534 .stop = frag_stop,
1535 .show = frag_show,
1536 };
1537
1538 static const struct seq_operations pagetypeinfo_op = {
1539 .start = frag_start,
1540 .next = frag_next,
1541 .stop = frag_stop,
1542 .show = pagetypeinfo_show,
1543 };
1544
1545 static bool is_zone_first_populated(pg_data_t *pgdat, struct zone *zone)
1546 {
1547 int zid;
1548
1549 for (zid = 0; zid < MAX_NR_ZONES; zid++) {
1550 struct zone *compare = &pgdat->node_zones[zid];
1551
1552 if (populated_zone(compare))
1553 return zone == compare;
1554 }
1555
1556 return false;
1557 }
1558
1559 static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat,
1560 struct zone *zone)
1561 {
1562 int i;
1563 seq_printf(m, "Node %d, zone %8s", pgdat->node_id, zone->name);
1564 if (is_zone_first_populated(pgdat, zone)) {
1565 seq_printf(m, "\n per-node stats");
1566 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) {
1567 seq_printf(m, "\n %-12s %lu",
1568 vmstat_text[i + NR_VM_ZONE_STAT_ITEMS +
1569 NR_VM_NUMA_STAT_ITEMS],
1570 node_page_state(pgdat, i));
1571 }
1572 }
1573 seq_printf(m,
1574 "\n pages free %lu"
1575 "\n min %lu"
1576 "\n low %lu"
1577 "\n high %lu"
1578 "\n spanned %lu"
1579 "\n present %lu"
1580 "\n managed %lu",
1581 zone_page_state(zone, NR_FREE_PAGES),
1582 min_wmark_pages(zone),
1583 low_wmark_pages(zone),
1584 high_wmark_pages(zone),
1585 zone->spanned_pages,
1586 zone->present_pages,
1587 zone_managed_pages(zone));
1588
1589 seq_printf(m,
1590 "\n protection: (%ld",
1591 zone->lowmem_reserve[0]);
1592 for (i = 1; i < ARRAY_SIZE(zone->lowmem_reserve); i++)
1593 seq_printf(m, ", %ld", zone->lowmem_reserve[i]);
1594 seq_putc(m, ')');
1595
1596
1597 if (!populated_zone(zone)) {
1598 seq_putc(m, '\n');
1599 return;
1600 }
1601
1602 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
1603 seq_printf(m, "\n %-12s %lu", vmstat_text[i],
1604 zone_page_state(zone, i));
1605
1606 #ifdef CONFIG_NUMA
1607 for (i = 0; i < NR_VM_NUMA_STAT_ITEMS; i++)
1608 seq_printf(m, "\n %-12s %lu",
1609 vmstat_text[i + NR_VM_ZONE_STAT_ITEMS],
1610 zone_numa_state_snapshot(zone, i));
1611 #endif
1612
1613 seq_printf(m, "\n pagesets");
1614 for_each_online_cpu(i) {
1615 struct per_cpu_pageset *pageset;
1616
1617 pageset = per_cpu_ptr(zone->pageset, i);
1618 seq_printf(m,
1619 "\n cpu: %i"
1620 "\n count: %i"
1621 "\n high: %i"
1622 "\n batch: %i",
1623 i,
1624 pageset->pcp.count,
1625 pageset->pcp.high,
1626 pageset->pcp.batch);
1627 #ifdef CONFIG_SMP
1628 seq_printf(m, "\n vm stats threshold: %d",
1629 pageset->stat_threshold);
1630 #endif
1631 }
1632 seq_printf(m,
1633 "\n node_unreclaimable: %u"
1634 "\n start_pfn: %lu",
1635 pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES,
1636 zone->zone_start_pfn);
1637 seq_putc(m, '\n');
1638 }
1639
1640
1641
1642
1643
1644
1645
1646 static int zoneinfo_show(struct seq_file *m, void *arg)
1647 {
1648 pg_data_t *pgdat = (pg_data_t *)arg;
1649 walk_zones_in_node(m, pgdat, false, false, zoneinfo_show_print);
1650 return 0;
1651 }
1652
1653 static const struct seq_operations zoneinfo_op = {
1654 .start = frag_start,
1655
1656 .next = frag_next,
1657 .stop = frag_stop,
1658 .show = zoneinfo_show,
1659 };
1660
1661 enum writeback_stat_item {
1662 NR_DIRTY_THRESHOLD,
1663 NR_DIRTY_BG_THRESHOLD,
1664 NR_VM_WRITEBACK_STAT_ITEMS,
1665 };
1666
1667 static void *vmstat_start(struct seq_file *m, loff_t *pos)
1668 {
1669 unsigned long *v;
1670 int i, stat_items_size;
1671
1672 if (*pos >= ARRAY_SIZE(vmstat_text))
1673 return NULL;
1674 stat_items_size = NR_VM_ZONE_STAT_ITEMS * sizeof(unsigned long) +
1675 NR_VM_NUMA_STAT_ITEMS * sizeof(unsigned long) +
1676 NR_VM_NODE_STAT_ITEMS * sizeof(unsigned long) +
1677 NR_VM_WRITEBACK_STAT_ITEMS * sizeof(unsigned long);
1678
1679 #ifdef CONFIG_VM_EVENT_COUNTERS
1680 stat_items_size += sizeof(struct vm_event_state);
1681 #endif
1682
1683 BUILD_BUG_ON(stat_items_size !=
1684 ARRAY_SIZE(vmstat_text) * sizeof(unsigned long));
1685 v = kmalloc(stat_items_size, GFP_KERNEL);
1686 m->private = v;
1687 if (!v)
1688 return ERR_PTR(-ENOMEM);
1689 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
1690 v[i] = global_zone_page_state(i);
1691 v += NR_VM_ZONE_STAT_ITEMS;
1692
1693 #ifdef CONFIG_NUMA
1694 for (i = 0; i < NR_VM_NUMA_STAT_ITEMS; i++)
1695 v[i] = global_numa_state(i);
1696 v += NR_VM_NUMA_STAT_ITEMS;
1697 #endif
1698
1699 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
1700 v[i] = global_node_page_state(i);
1701 v += NR_VM_NODE_STAT_ITEMS;
1702
1703 global_dirty_limits(v + NR_DIRTY_BG_THRESHOLD,
1704 v + NR_DIRTY_THRESHOLD);
1705 v += NR_VM_WRITEBACK_STAT_ITEMS;
1706
1707 #ifdef CONFIG_VM_EVENT_COUNTERS
1708 all_vm_events(v);
1709 v[PGPGIN] /= 2;
1710 v[PGPGOUT] /= 2;
1711 #endif
1712 return (unsigned long *)m->private + *pos;
1713 }
1714
1715 static void *vmstat_next(struct seq_file *m, void *arg, loff_t *pos)
1716 {
1717 (*pos)++;
1718 if (*pos >= ARRAY_SIZE(vmstat_text))
1719 return NULL;
1720 return (unsigned long *)m->private + *pos;
1721 }
1722
1723 static int vmstat_show(struct seq_file *m, void *arg)
1724 {
1725 unsigned long *l = arg;
1726 unsigned long off = l - (unsigned long *)m->private;
1727
1728 seq_puts(m, vmstat_text[off]);
1729 seq_put_decimal_ull(m, " ", *l);
1730 seq_putc(m, '\n');
1731 return 0;
1732 }
1733
1734 static void vmstat_stop(struct seq_file *m, void *arg)
1735 {
1736 kfree(m->private);
1737 m->private = NULL;
1738 }
1739
1740 static const struct seq_operations vmstat_op = {
1741 .start = vmstat_start,
1742 .next = vmstat_next,
1743 .stop = vmstat_stop,
1744 .show = vmstat_show,
1745 };
1746 #endif
1747
1748 #ifdef CONFIG_SMP
1749 static DEFINE_PER_CPU(struct delayed_work, vmstat_work);
1750 int sysctl_stat_interval __read_mostly = HZ;
1751
1752 #ifdef CONFIG_PROC_FS
1753 static void refresh_vm_stats(struct work_struct *work)
1754 {
1755 refresh_cpu_vm_stats(true);
1756 }
1757
1758 int vmstat_refresh(struct ctl_table *table, int write,
1759 void __user *buffer, size_t *lenp, loff_t *ppos)
1760 {
1761 long val;
1762 int err;
1763 int i;
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777 err = schedule_on_each_cpu(refresh_vm_stats);
1778 if (err)
1779 return err;
1780 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) {
1781 val = atomic_long_read(&vm_zone_stat[i]);
1782 if (val < 0) {
1783 pr_warn("%s: %s %ld\n",
1784 __func__, vmstat_text[i], val);
1785 err = -EINVAL;
1786 }
1787 }
1788 #ifdef CONFIG_NUMA
1789 for (i = 0; i < NR_VM_NUMA_STAT_ITEMS; i++) {
1790 val = atomic_long_read(&vm_numa_stat[i]);
1791 if (val < 0) {
1792 pr_warn("%s: %s %ld\n",
1793 __func__, vmstat_text[i + NR_VM_ZONE_STAT_ITEMS], val);
1794 err = -EINVAL;
1795 }
1796 }
1797 #endif
1798 if (err)
1799 return err;
1800 if (write)
1801 *ppos += *lenp;
1802 else
1803 *lenp = 0;
1804 return 0;
1805 }
1806 #endif
1807
1808 static void vmstat_update(struct work_struct *w)
1809 {
1810 if (refresh_cpu_vm_stats(true)) {
1811
1812
1813
1814
1815
1816 queue_delayed_work_on(smp_processor_id(), mm_percpu_wq,
1817 this_cpu_ptr(&vmstat_work),
1818 round_jiffies_relative(sysctl_stat_interval));
1819 }
1820 }
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831 static bool need_update(int cpu)
1832 {
1833 struct zone *zone;
1834
1835 for_each_populated_zone(zone) {
1836 struct per_cpu_pageset *p = per_cpu_ptr(zone->pageset, cpu);
1837
1838 BUILD_BUG_ON(sizeof(p->vm_stat_diff[0]) != 1);
1839 #ifdef CONFIG_NUMA
1840 BUILD_BUG_ON(sizeof(p->vm_numa_stat_diff[0]) != 2);
1841 #endif
1842
1843
1844
1845
1846 if (memchr_inv(p->vm_stat_diff, 0, NR_VM_ZONE_STAT_ITEMS *
1847 sizeof(p->vm_stat_diff[0])))
1848 return true;
1849 #ifdef CONFIG_NUMA
1850 if (memchr_inv(p->vm_numa_stat_diff, 0, NR_VM_NUMA_STAT_ITEMS *
1851 sizeof(p->vm_numa_stat_diff[0])))
1852 return true;
1853 #endif
1854 }
1855 return false;
1856 }
1857
1858
1859
1860
1861
1862
1863 void quiet_vmstat(void)
1864 {
1865 if (system_state != SYSTEM_RUNNING)
1866 return;
1867
1868 if (!delayed_work_pending(this_cpu_ptr(&vmstat_work)))
1869 return;
1870
1871 if (!need_update(smp_processor_id()))
1872 return;
1873
1874
1875
1876
1877
1878
1879
1880 refresh_cpu_vm_stats(false);
1881 }
1882
1883
1884
1885
1886
1887
1888
1889 static void vmstat_shepherd(struct work_struct *w);
1890
1891 static DECLARE_DEFERRABLE_WORK(shepherd, vmstat_shepherd);
1892
1893 static void vmstat_shepherd(struct work_struct *w)
1894 {
1895 int cpu;
1896
1897 get_online_cpus();
1898
1899 for_each_online_cpu(cpu) {
1900 struct delayed_work *dw = &per_cpu(vmstat_work, cpu);
1901
1902 if (!delayed_work_pending(dw) && need_update(cpu))
1903 queue_delayed_work_on(cpu, mm_percpu_wq, dw, 0);
1904 }
1905 put_online_cpus();
1906
1907 schedule_delayed_work(&shepherd,
1908 round_jiffies_relative(sysctl_stat_interval));
1909 }
1910
1911 static void __init start_shepherd_timer(void)
1912 {
1913 int cpu;
1914
1915 for_each_possible_cpu(cpu)
1916 INIT_DEFERRABLE_WORK(per_cpu_ptr(&vmstat_work, cpu),
1917 vmstat_update);
1918
1919 schedule_delayed_work(&shepherd,
1920 round_jiffies_relative(sysctl_stat_interval));
1921 }
1922
1923 static void __init init_cpu_node_state(void)
1924 {
1925 int node;
1926
1927 for_each_online_node(node) {
1928 if (cpumask_weight(cpumask_of_node(node)) > 0)
1929 node_set_state(node, N_CPU);
1930 }
1931 }
1932
1933 static int vmstat_cpu_online(unsigned int cpu)
1934 {
1935 refresh_zone_stat_thresholds();
1936 node_set_state(cpu_to_node(cpu), N_CPU);
1937 return 0;
1938 }
1939
1940 static int vmstat_cpu_down_prep(unsigned int cpu)
1941 {
1942 cancel_delayed_work_sync(&per_cpu(vmstat_work, cpu));
1943 return 0;
1944 }
1945
1946 static int vmstat_cpu_dead(unsigned int cpu)
1947 {
1948 const struct cpumask *node_cpus;
1949 int node;
1950
1951 node = cpu_to_node(cpu);
1952
1953 refresh_zone_stat_thresholds();
1954 node_cpus = cpumask_of_node(node);
1955 if (cpumask_weight(node_cpus) > 0)
1956 return 0;
1957
1958 node_clear_state(node, N_CPU);
1959 return 0;
1960 }
1961
1962 #endif
1963
1964 struct workqueue_struct *mm_percpu_wq;
1965
1966 void __init init_mm_internals(void)
1967 {
1968 int ret __maybe_unused;
1969
1970 mm_percpu_wq = alloc_workqueue("mm_percpu_wq", WQ_MEM_RECLAIM, 0);
1971
1972 #ifdef CONFIG_SMP
1973 ret = cpuhp_setup_state_nocalls(CPUHP_MM_VMSTAT_DEAD, "mm/vmstat:dead",
1974 NULL, vmstat_cpu_dead);
1975 if (ret < 0)
1976 pr_err("vmstat: failed to register 'dead' hotplug state\n");
1977
1978 ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "mm/vmstat:online",
1979 vmstat_cpu_online,
1980 vmstat_cpu_down_prep);
1981 if (ret < 0)
1982 pr_err("vmstat: failed to register 'online' hotplug state\n");
1983
1984 get_online_cpus();
1985 init_cpu_node_state();
1986 put_online_cpus();
1987
1988 start_shepherd_timer();
1989 #endif
1990 #ifdef CONFIG_PROC_FS
1991 proc_create_seq("buddyinfo", 0444, NULL, &fragmentation_op);
1992 proc_create_seq("pagetypeinfo", 0400, NULL, &pagetypeinfo_op);
1993 proc_create_seq("vmstat", 0444, NULL, &vmstat_op);
1994 proc_create_seq("zoneinfo", 0444, NULL, &zoneinfo_op);
1995 #endif
1996 }
1997
1998 #if defined(CONFIG_DEBUG_FS) && defined(CONFIG_COMPACTION)
1999
2000
2001
2002
2003
2004 static int unusable_free_index(unsigned int order,
2005 struct contig_page_info *info)
2006 {
2007
2008 if (info->free_pages == 0)
2009 return 1000;
2010
2011
2012
2013
2014
2015
2016
2017
2018 return div_u64((info->free_pages - (info->free_blocks_suitable << order)) * 1000ULL, info->free_pages);
2019
2020 }
2021
2022 static void unusable_show_print(struct seq_file *m,
2023 pg_data_t *pgdat, struct zone *zone)
2024 {
2025 unsigned int order;
2026 int index;
2027 struct contig_page_info info;
2028
2029 seq_printf(m, "Node %d, zone %8s ",
2030 pgdat->node_id,
2031 zone->name);
2032 for (order = 0; order < MAX_ORDER; ++order) {
2033 fill_contig_page_info(zone, order, &info);
2034 index = unusable_free_index(order, &info);
2035 seq_printf(m, "%d.%03d ", index / 1000, index % 1000);
2036 }
2037
2038 seq_putc(m, '\n');
2039 }
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050 static int unusable_show(struct seq_file *m, void *arg)
2051 {
2052 pg_data_t *pgdat = (pg_data_t *)arg;
2053
2054
2055 if (!node_state(pgdat->node_id, N_MEMORY))
2056 return 0;
2057
2058 walk_zones_in_node(m, pgdat, true, false, unusable_show_print);
2059
2060 return 0;
2061 }
2062
2063 static const struct seq_operations unusable_op = {
2064 .start = frag_start,
2065 .next = frag_next,
2066 .stop = frag_stop,
2067 .show = unusable_show,
2068 };
2069
2070 static int unusable_open(struct inode *inode, struct file *file)
2071 {
2072 return seq_open(file, &unusable_op);
2073 }
2074
2075 static const struct file_operations unusable_file_ops = {
2076 .open = unusable_open,
2077 .read = seq_read,
2078 .llseek = seq_lseek,
2079 .release = seq_release,
2080 };
2081
2082 static void extfrag_show_print(struct seq_file *m,
2083 pg_data_t *pgdat, struct zone *zone)
2084 {
2085 unsigned int order;
2086 int index;
2087
2088
2089 struct contig_page_info info;
2090
2091 seq_printf(m, "Node %d, zone %8s ",
2092 pgdat->node_id,
2093 zone->name);
2094 for (order = 0; order < MAX_ORDER; ++order) {
2095 fill_contig_page_info(zone, order, &info);
2096 index = __fragmentation_index(order, &info);
2097 seq_printf(m, "%d.%03d ", index / 1000, index % 1000);
2098 }
2099
2100 seq_putc(m, '\n');
2101 }
2102
2103
2104
2105
2106 static int extfrag_show(struct seq_file *m, void *arg)
2107 {
2108 pg_data_t *pgdat = (pg_data_t *)arg;
2109
2110 walk_zones_in_node(m, pgdat, true, false, extfrag_show_print);
2111
2112 return 0;
2113 }
2114
2115 static const struct seq_operations extfrag_op = {
2116 .start = frag_start,
2117 .next = frag_next,
2118 .stop = frag_stop,
2119 .show = extfrag_show,
2120 };
2121
2122 static int extfrag_open(struct inode *inode, struct file *file)
2123 {
2124 return seq_open(file, &extfrag_op);
2125 }
2126
2127 static const struct file_operations extfrag_file_ops = {
2128 .open = extfrag_open,
2129 .read = seq_read,
2130 .llseek = seq_lseek,
2131 .release = seq_release,
2132 };
2133
2134 static int __init extfrag_debug_init(void)
2135 {
2136 struct dentry *extfrag_debug_root;
2137
2138 extfrag_debug_root = debugfs_create_dir("extfrag", NULL);
2139
2140 debugfs_create_file("unusable_index", 0444, extfrag_debug_root, NULL,
2141 &unusable_file_ops);
2142
2143 debugfs_create_file("extfrag_index", 0444, extfrag_debug_root, NULL,
2144 &extfrag_file_ops);
2145
2146 return 0;
2147 }
2148
2149 module_init(extfrag_debug_init);
2150 #endif