319 static int _alloc_read_4_write(struct ore_io_state *ios)  in _alloc_read_4_write()  argument
321 	struct ore_layout *layout = ios->layout;  in _alloc_read_4_write()
326 	unsigned sgs_per_dev = ios->sp2d->pages_in_unit + 2;  in _alloc_read_4_write()
328 	ret = _ore_get_io_state(layout, ios->oc,  in _alloc_read_4_write()
330 				sgs_per_dev, 0, &ios->ios_read_4_write);  in _alloc_read_4_write()
337 static int _add_to_r4w(struct ore_io_state *ios, struct ore_striping_info *si,  in _add_to_r4w()  argument
344 			  (ios->layout->group_width * ios->layout->mirrors_p1));  in _add_to_r4w()
348 	if (!ios->ios_read_4_write) {  in _add_to_r4w()
349 		int ret = _alloc_read_4_write(ios);  in _add_to_r4w()
355 	read_ios = ios->ios_read_4_write;  in _add_to_r4w()
356 	read_ios->numdevs = ios->layout->group_width * ios->layout->mirrors_p1;  in _add_to_r4w()
361 					   ios->sp2d->pages_in_unit);  in _add_to_r4w()
364 				     ios->sp2d->pages_in_unit);  in _add_to_r4w()
388 static int _add_to_r4w_first_page(struct ore_io_state *ios, struct page *page)  in _add_to_r4w_first_page()  argument
393 	ore_calc_stripe_info(ios->layout, ios->offset, 0, &si);  in _add_to_r4w_first_page()
401 	return _add_to_r4w(ios, &si, page, pg_len);  in _add_to_r4w_first_page()
405 static int _add_to_r4w_last_page(struct ore_io_state *ios, u64 *offset)  in _add_to_r4w_last_page()  argument
411 	ore_calc_stripe_info(ios->layout, *offset, 0, &si);  in _add_to_r4w_last_page()
415 	page = ios->sp2d->_1p_stripes[p].pages[c];  in _add_to_r4w_last_page()
425 	return _add_to_r4w(ios, &si, page, pg_len);  in _add_to_r4w_last_page()
428 static void _mark_read4write_pages_uptodate(struct ore_io_state *ios, int ret)  in _mark_read4write_pages_uptodate()  argument
434 	for (d = 0; d < ios->numdevs; d++) {  in _mark_read4write_pages_uptodate()
435 		struct bio *bio = ios->per_dev[d].bio;  in _mark_read4write_pages_uptodate()
469 static int _read_4_write_first_stripe(struct ore_io_state *ios)  in _read_4_write_first_stripe()  argument
472 	struct __stripe_pages_2d *sp2d = ios->sp2d;  in _read_4_write_first_stripe()
473 	u64 offset = ios->si.first_stripe_start;  in _read_4_write_first_stripe()
476 	if (offset == ios->offset) /* Go to start collect $200 */  in _read_4_write_first_stripe()
483 		   offset, ios->offset, min_p, max_p);  in _read_4_write_first_stripe()
486 		ore_calc_stripe_info(ios->layout, offset, 0, &read_si);  in _read_4_write_first_stripe()
495 				if (ios->offset % PAGE_SIZE)  in _read_4_write_first_stripe()
497 					_add_to_r4w_first_page(ios, *pp);  in _read_4_write_first_stripe()
502 			*pp = ios->r4w->get_page(ios->private, offset,  in _read_4_write_first_stripe()
508 				_add_to_r4w(ios, &read_si, *pp, PAGE_SIZE);  in _read_4_write_first_stripe()
522 static int _read_4_write_last_stripe(struct ore_io_state *ios)  in _read_4_write_last_stripe()  argument
525 	struct __stripe_pages_2d *sp2d = ios->sp2d;  in _read_4_write_last_stripe()
528 	unsigned bytes_in_stripe = ios->si.bytes_in_stripe;  in _read_4_write_last_stripe()
531 	offset = ios->offset + ios->length;  in _read_4_write_last_stripe()
533 		_add_to_r4w_last_page(ios, &offset);  in _read_4_write_last_stripe()
541 	ore_calc_stripe_info(ios->layout, offset, 0, &read_si);  in _read_4_write_last_stripe()
562 			page = ios->r4w->get_page(ios->private, offset,  in _read_4_write_last_stripe()
571 				_add_to_r4w(ios, &read_si, page, PAGE_SIZE);  in _read_4_write_last_stripe()
578 			ore_calc_stripe_info(ios->layout, offset, 0, &read_si);  in _read_4_write_last_stripe()
589 static int _read_4_write_execute(struct ore_io_state *ios)  in _read_4_write_execute()  argument
595 	ios_read = ios->ios_read_4_write;  in _read_4_write_execute()
602 	ios_read->pages = ios->pages;  in _read_4_write_execute()
619 	ios->ios_read_4_write = NULL; /* Might need a reuse at last stripe */  in _read_4_write_execute()
624 int _ore_add_parity_unit(struct ore_io_state *ios,  in _ore_add_parity_unit()  argument
629 	if (ios->reading) {  in _ore_add_parity_unit()
630 		if (per_dev->cur_sg >= ios->sgs_per_dev) {  in _ore_add_parity_unit()
632 				per_dev->cur_sg, ios->sgs_per_dev);  in _ore_add_parity_unit()
637 		struct __stripe_pages_2d *sp2d = ios->sp2d;  in _ore_add_parity_unit()
638 		struct page **pages = ios->parity_pages + ios->cur_par_page;  in _ore_add_parity_unit()
653 				_read_4_write_first_stripe(ios);  in _ore_add_parity_unit()
657 			_read_4_write_last_stripe(ios);  in _ore_add_parity_unit()
658 		_read_4_write_execute(ios);  in _ore_add_parity_unit()
665 			++(ios->cur_par_page);  in _ore_add_parity_unit()
671 		ret = _ore_add_stripe_unit(ios,  &array_start, 0, pages,  in _ore_add_parity_unit()
678 			_sp2d_reset(sp2d, ios->r4w, ios->private);  in _ore_add_parity_unit()
684 int _ore_post_alloc_raid_stuff(struct ore_io_state *ios)  in _ore_post_alloc_raid_stuff()  argument
686 	if (ios->parity_pages) {  in _ore_post_alloc_raid_stuff()
687 		struct ore_layout *layout = ios->layout;  in _ore_post_alloc_raid_stuff()
691 				layout->parity, &ios->sp2d)) {  in _ore_post_alloc_raid_stuff()
698 void _ore_free_raid_stuff(struct ore_io_state *ios)  in _ore_free_raid_stuff()  argument
700 	if (ios->sp2d) { /* writing and raid */  in _ore_free_raid_stuff()
703 		for (i = 0; i < ios->cur_par_page; i++) {  in _ore_free_raid_stuff()
704 			struct page *page = ios->parity_pages[i];  in _ore_free_raid_stuff()
709 		if (ios->extra_part_alloc)  in _ore_free_raid_stuff()
710 			kfree(ios->parity_pages);  in _ore_free_raid_stuff()
712 		_sp2d_reset(ios->sp2d, ios->r4w, ios->private);  in _ore_free_raid_stuff()
713 		_sp2d_free(ios->sp2d);  in _ore_free_raid_stuff()
716 		if (ios->extra_part_alloc)  in _ore_free_raid_stuff()
717 			kfree(ios->per_dev[0].sglist);  in _ore_free_raid_stuff()
719 	if (ios->ios_read_4_write)  in _ore_free_raid_stuff()
720 		ore_put_io_state(ios->ios_read_4_write);  in _ore_free_raid_stuff()