172 	struct dm_stat *s = container_of(head, struct dm_stat, rcu_head);  in dm_stat_free()  local
174 	kfree(s->program_id);  in dm_stat_free()
175 	kfree(s->aux_data);  in dm_stat_free()
177 		dm_kvfree(s->stat_percpu[cpu], s->percpu_alloc_size);  in dm_stat_free()
178 	dm_kvfree(s, s->shared_alloc_size);  in dm_stat_free()
205 	struct dm_stat *s;  in dm_stats_cleanup()  local
209 		s = container_of(stats->list.next, struct dm_stat, list_entry);  in dm_stats_cleanup()
210 		list_del(&s->list_entry);  in dm_stats_cleanup()
211 		for (ni = 0; ni < s->n_entries; ni++) {  in dm_stats_cleanup()
212 			shared = &s->stat_shared[ni];  in dm_stats_cleanup()
217 				       (unsigned long long)s->start,  in dm_stats_cleanup()
218 				       (unsigned long long)s->end,  in dm_stats_cleanup()
219 				       (unsigned long long)s->step,  in dm_stats_cleanup()
224 		dm_stat_free(&s->rcu_head);  in dm_stats_cleanup()
236 	struct dm_stat *s, *tmp_s;  in dm_stats_create()  local
267 	s = dm_kvzalloc(shared_alloc_size, NUMA_NO_NODE);  in dm_stats_create()
268 	if (!s)  in dm_stats_create()
271 	s->n_entries = n_entries;  in dm_stats_create()
272 	s->start = start;  in dm_stats_create()
273 	s->end = end;  in dm_stats_create()
274 	s->step = step;  in dm_stats_create()
275 	s->shared_alloc_size = shared_alloc_size;  in dm_stats_create()
276 	s->percpu_alloc_size = percpu_alloc_size;  in dm_stats_create()
278 	s->program_id = kstrdup(program_id, GFP_KERNEL);  in dm_stats_create()
279 	if (!s->program_id) {  in dm_stats_create()
283 	s->aux_data = kstrdup(aux_data, GFP_KERNEL);  in dm_stats_create()
284 	if (!s->aux_data) {  in dm_stats_create()
290 		atomic_set(&s->stat_shared[ni].in_flight[READ], 0);  in dm_stats_create()
291 		atomic_set(&s->stat_shared[ni].in_flight[WRITE], 0);  in dm_stats_create()
300 		s->stat_percpu[cpu] = p;  in dm_stats_create()
313 	s->id = 0;  in dm_stats_create()
316 		if (WARN_ON(tmp_s->id < s->id)) {  in dm_stats_create()
320 		if (tmp_s->id > s->id)  in dm_stats_create()
322 		if (unlikely(s->id == INT_MAX)) {  in dm_stats_create()
326 		s->id++;  in dm_stats_create()
328 	ret_id = s->id;  in dm_stats_create()
329 	list_add_tail_rcu(&s->list_entry, l);  in dm_stats_create()
340 	dm_stat_free(&s->rcu_head);  in dm_stats_create()
346 	struct dm_stat *s;  in __dm_stats_find()  local
348 	list_for_each_entry(s, &stats->list, list_entry) {  in __dm_stats_find()
349 		if (s->id > id)  in __dm_stats_find()
351 		if (s->id == id)  in __dm_stats_find()
352 			return s;  in __dm_stats_find()
360 	struct dm_stat *s;  in dm_stats_delete()  local
365 	s = __dm_stats_find(stats, id);  in dm_stats_delete()
366 	if (!s) {  in dm_stats_delete()
371 	list_del_rcu(&s->list_entry);  in dm_stats_delete()
378 		if (is_vmalloc_addr(s->stat_percpu))  in dm_stats_delete()
380 	if (is_vmalloc_addr(s)) {  in dm_stats_delete()
383 		dm_stat_free(&s->rcu_head);  in dm_stats_delete()
386 		call_rcu(&s->rcu_head, dm_stat_free);  in dm_stats_delete()
394 	struct dm_stat *s;  in dm_stats_list()  local
404 	list_for_each_entry(s, &stats->list, list_entry) {  in dm_stats_list()
405 		if (!program || !strcmp(program, s->program_id)) {  in dm_stats_list()
406 			len = s->end - s->start;  in dm_stats_list()
407 			DMEMIT("%d: %llu+%llu %llu %s %s\n", s->id,  in dm_stats_list()
408 				(unsigned long long)s->start,  in dm_stats_list()
410 				(unsigned long long)s->step,  in dm_stats_list()
411 				s->program_id,  in dm_stats_list()
412 				s->aux_data);  in dm_stats_list()
445 static void dm_stat_for_entry(struct dm_stat *s, size_t entry,  in dm_stat_for_entry()  argument
450 	struct dm_stat_shared *shared = &s->stat_shared[entry];  in dm_stat_for_entry()
474 	p = &s->stat_percpu[smp_processor_id()][entry];  in dm_stat_for_entry()
495 static void __dm_stat_bio(struct dm_stat *s, unsigned long bi_rw,  in __dm_stat_bio()  argument
503 	if (end_sector <= s->start || bi_sector >= s->end)  in __dm_stat_bio()
505 	if (unlikely(bi_sector < s->start)) {  in __dm_stat_bio()
507 		todo = end_sector - s->start;  in __dm_stat_bio()
509 		rel_sector = bi_sector - s->start;  in __dm_stat_bio()
512 	if (unlikely(end_sector > s->end))  in __dm_stat_bio()
513 		todo -= (end_sector - s->end);  in __dm_stat_bio()
515 	offset = dm_sector_div64(rel_sector, s->step);  in __dm_stat_bio()
518 		if (WARN_ON_ONCE(entry >= s->n_entries)) {  in __dm_stat_bio()
519 			DMCRIT("Invalid area access in region id %d", s->id);  in __dm_stat_bio()
523 		if (fragment_len > s->step - offset)  in __dm_stat_bio()
524 			fragment_len = s->step - offset;  in __dm_stat_bio()
525 		dm_stat_for_entry(s, entry, bi_rw, fragment_len,  in __dm_stat_bio()
537 	struct dm_stat *s;  in dm_stats_account_io()  local
563 	list_for_each_entry_rcu(s, &stats->list, list_entry)  in dm_stats_account_io()
564 		__dm_stat_bio(s, bi_rw, bi_sector, end_sector, end, duration, stats_aux);  in dm_stats_account_io()
570 						   struct dm_stat *s, size_t x)  in __dm_stat_init_temporary_percpu_totals()  argument
576 	p = &s->stat_percpu[smp_processor_id()][x];  in __dm_stat_init_temporary_percpu_totals()
582 		p = &s->stat_percpu[cpu][x];  in __dm_stat_init_temporary_percpu_totals()
598 static void __dm_stat_clear(struct dm_stat *s, size_t idx_start, size_t idx_end,  in __dm_stat_clear()  argument
606 		shared = &s->stat_shared[x];  in __dm_stat_clear()
608 			__dm_stat_init_temporary_percpu_totals(shared, s, x);  in __dm_stat_clear()
610 		p = &s->stat_percpu[smp_processor_id()][x];  in __dm_stat_clear()
629 	struct dm_stat *s;  in dm_stats_clear()  local
633 	s = __dm_stats_find(stats, id);  in dm_stats_clear()
634 	if (!s) {  in dm_stats_clear()
639 	__dm_stat_clear(s, 0, s->n_entries, true);  in dm_stats_clear()
671 	struct dm_stat *s;  in dm_stats_print()  local
684 	s = __dm_stats_find(stats, id);  in dm_stats_print()
685 	if (!s) {  in dm_stats_print()
692 	    idx_end > s->n_entries)  in dm_stats_print()
693 		idx_end = s->n_entries;  in dm_stats_print()
698 	step = s->step;  in dm_stats_print()
699 	start = s->start + (step * idx_start);  in dm_stats_print()
702 		shared = &s->stat_shared[x];  in dm_stats_print()
704 		if (unlikely(end > s->end))  in dm_stats_print()
705 			end = s->end;  in dm_stats_print()
707 		__dm_stat_init_temporary_percpu_totals(shared, s, x);  in dm_stats_print()
731 		__dm_stat_clear(s, idx_start, idx_end, false);  in dm_stats_print()
741 	struct dm_stat *s;  in dm_stats_set_aux()  local
746 	s = __dm_stats_find(stats, id);  in dm_stats_set_aux()
747 	if (!s) {  in dm_stats_set_aux()
758 	kfree(s->aux_data);  in dm_stats_set_aux()
759 	s->aux_data = new_aux_data;  in dm_stats_set_aux()