181 static int dm_stat_in_flight(struct dm_stat_shared *shared)  in dm_stat_in_flight()  argument
183 	return atomic_read(&shared->in_flight[READ]) +  in dm_stat_in_flight()
184 	       atomic_read(&shared->in_flight[WRITE]);  in dm_stat_in_flight()
206 	struct dm_stat_shared *shared;  in dm_stats_cleanup()  local
212 			shared = &s->stat_shared[ni];  in dm_stats_cleanup()
213 			if (WARN_ON(dm_stat_in_flight(shared))) {  in dm_stats_cleanup()
220 				       atomic_read(&shared->in_flight[READ]),  in dm_stats_cleanup()
221 				       atomic_read(&shared->in_flight[WRITE]));  in dm_stats_cleanup()
420 static void dm_stat_round(struct dm_stat_shared *shared, struct dm_stat_percpu *p)  in dm_stat_round()  argument
428 	unsigned long difference = now - shared->stamp;  in dm_stat_round()
432 	in_flight_read = (unsigned)atomic_read(&shared->in_flight[READ]);  in dm_stat_round()
433 	in_flight_write = (unsigned)atomic_read(&shared->in_flight[WRITE]);  in dm_stat_round()
442 	shared->stamp = now;  in dm_stat_round()
450 	struct dm_stat_shared *shared = &s->stat_shared[entry];  in dm_stat_for_entry()  local
477 		dm_stat_round(shared, p);  in dm_stat_for_entry()
478 		atomic_inc(&shared->in_flight[idx]);  in dm_stat_for_entry()
480 		dm_stat_round(shared, p);  in dm_stat_for_entry()
481 		atomic_dec(&shared->in_flight[idx]);  in dm_stat_for_entry()
569 static void __dm_stat_init_temporary_percpu_totals(struct dm_stat_shared *shared,  in __dm_stat_init_temporary_percpu_totals()  argument
577 	dm_stat_round(shared, p);  in __dm_stat_init_temporary_percpu_totals()
580 	memset(&shared->tmp, 0, sizeof(shared->tmp));  in __dm_stat_init_temporary_percpu_totals()
583 		shared->tmp.sectors[READ] += ACCESS_ONCE(p->sectors[READ]);  in __dm_stat_init_temporary_percpu_totals()
584 		shared->tmp.sectors[WRITE] += ACCESS_ONCE(p->sectors[WRITE]);  in __dm_stat_init_temporary_percpu_totals()
585 		shared->tmp.ios[READ] += ACCESS_ONCE(p->ios[READ]);  in __dm_stat_init_temporary_percpu_totals()
586 		shared->tmp.ios[WRITE] += ACCESS_ONCE(p->ios[WRITE]);  in __dm_stat_init_temporary_percpu_totals()
587 		shared->tmp.merges[READ] += ACCESS_ONCE(p->merges[READ]);  in __dm_stat_init_temporary_percpu_totals()
588 		shared->tmp.merges[WRITE] += ACCESS_ONCE(p->merges[WRITE]);  in __dm_stat_init_temporary_percpu_totals()
589 		shared->tmp.ticks[READ] += ACCESS_ONCE(p->ticks[READ]);  in __dm_stat_init_temporary_percpu_totals()
590 		shared->tmp.ticks[WRITE] += ACCESS_ONCE(p->ticks[WRITE]);  in __dm_stat_init_temporary_percpu_totals()
591 		shared->tmp.io_ticks[READ] += ACCESS_ONCE(p->io_ticks[READ]);  in __dm_stat_init_temporary_percpu_totals()
592 		shared->tmp.io_ticks[WRITE] += ACCESS_ONCE(p->io_ticks[WRITE]);  in __dm_stat_init_temporary_percpu_totals()
593 		shared->tmp.io_ticks_total += ACCESS_ONCE(p->io_ticks_total);  in __dm_stat_init_temporary_percpu_totals()
594 		shared->tmp.time_in_queue += ACCESS_ONCE(p->time_in_queue);  in __dm_stat_init_temporary_percpu_totals()
602 	struct dm_stat_shared *shared;  in __dm_stat_clear()  local
606 		shared = &s->stat_shared[x];  in __dm_stat_clear()
608 			__dm_stat_init_temporary_percpu_totals(shared, s, x);  in __dm_stat_clear()
611 		p->sectors[READ] -= shared->tmp.sectors[READ];  in __dm_stat_clear()
612 		p->sectors[WRITE] -= shared->tmp.sectors[WRITE];  in __dm_stat_clear()
613 		p->ios[READ] -= shared->tmp.ios[READ];  in __dm_stat_clear()
614 		p->ios[WRITE] -= shared->tmp.ios[WRITE];  in __dm_stat_clear()
615 		p->merges[READ] -= shared->tmp.merges[READ];  in __dm_stat_clear()
616 		p->merges[WRITE] -= shared->tmp.merges[WRITE];  in __dm_stat_clear()
617 		p->ticks[READ] -= shared->tmp.ticks[READ];  in __dm_stat_clear()
618 		p->ticks[WRITE] -= shared->tmp.ticks[WRITE];  in __dm_stat_clear()
619 		p->io_ticks[READ] -= shared->tmp.io_ticks[READ];  in __dm_stat_clear()
620 		p->io_ticks[WRITE] -= shared->tmp.io_ticks[WRITE];  in __dm_stat_clear()
621 		p->io_ticks_total -= shared->tmp.io_ticks_total;  in __dm_stat_clear()
622 		p->time_in_queue -= shared->tmp.time_in_queue;  in __dm_stat_clear()
675 	struct dm_stat_shared *shared;  in dm_stats_print()  local
702 		shared = &s->stat_shared[x];  in dm_stats_print()
707 		__dm_stat_init_temporary_percpu_totals(shared, s, x);  in dm_stats_print()
712 		       shared->tmp.ios[READ],  in dm_stats_print()
713 		       shared->tmp.merges[READ],  in dm_stats_print()
714 		       shared->tmp.sectors[READ],  in dm_stats_print()
715 		       dm_jiffies_to_msec64(shared->tmp.ticks[READ]),  in dm_stats_print()
716 		       shared->tmp.ios[WRITE],  in dm_stats_print()
717 		       shared->tmp.merges[WRITE],  in dm_stats_print()
718 		       shared->tmp.sectors[WRITE],  in dm_stats_print()
719 		       dm_jiffies_to_msec64(shared->tmp.ticks[WRITE]),  in dm_stats_print()
720 		       dm_stat_in_flight(shared),  in dm_stats_print()
721 		       dm_jiffies_to_msec64(shared->tmp.io_ticks_total),  in dm_stats_print()
722 		       dm_jiffies_to_msec64(shared->tmp.time_in_queue),  in dm_stats_print()
723 		       dm_jiffies_to_msec64(shared->tmp.io_ticks[READ]),  in dm_stats_print()
724 		       dm_jiffies_to_msec64(shared->tmp.io_ticks[WRITE]));  in dm_stats_print()