828 static void ceph_msg_data_bio_cursor_init(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_bio_cursor_init()  argument
831 	struct ceph_msg_data *data = cursor->data;  in ceph_msg_data_bio_cursor_init()
839 	cursor->resid = min(length, data->bio_length);  in ceph_msg_data_bio_cursor_init()
840 	cursor->bio = bio;  in ceph_msg_data_bio_cursor_init()
841 	cursor->bvec_iter = bio->bi_iter;  in ceph_msg_data_bio_cursor_init()
842 	cursor->last_piece =  in ceph_msg_data_bio_cursor_init()
843 		cursor->resid <= bio_iter_len(bio, cursor->bvec_iter);  in ceph_msg_data_bio_cursor_init()
846 static struct page *ceph_msg_data_bio_next(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_bio_next()  argument
850 	struct ceph_msg_data *data = cursor->data;  in ceph_msg_data_bio_next()
856 	bio = cursor->bio;  in ceph_msg_data_bio_next()
859 	bio_vec = bio_iter_iovec(bio, cursor->bvec_iter);  in ceph_msg_data_bio_next()
863 	if (cursor->last_piece) /* pagelist offset is always 0 */  in ceph_msg_data_bio_next()
864 		*length = cursor->resid;  in ceph_msg_data_bio_next()
867 	BUG_ON(*length > cursor->resid);  in ceph_msg_data_bio_next()
873 static bool ceph_msg_data_bio_advance(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_bio_advance()  argument
879 	BUG_ON(cursor->data->type != CEPH_MSG_DATA_BIO);  in ceph_msg_data_bio_advance()
881 	bio = cursor->bio;  in ceph_msg_data_bio_advance()
884 	bio_vec = bio_iter_iovec(bio, cursor->bvec_iter);  in ceph_msg_data_bio_advance()
888 	BUG_ON(cursor->resid < bytes);  in ceph_msg_data_bio_advance()
889 	cursor->resid -= bytes;  in ceph_msg_data_bio_advance()
891 	bio_advance_iter(bio, &cursor->bvec_iter, bytes);  in ceph_msg_data_bio_advance()
898 	if (!cursor->bvec_iter.bi_size) {  in ceph_msg_data_bio_advance()
900 		cursor->bio = bio;  in ceph_msg_data_bio_advance()
902 			cursor->bvec_iter = bio->bi_iter;  in ceph_msg_data_bio_advance()
904 			memset(&cursor->bvec_iter, 0,  in ceph_msg_data_bio_advance()
905 			       sizeof(cursor->bvec_iter));  in ceph_msg_data_bio_advance()
908 	if (!cursor->last_piece) {  in ceph_msg_data_bio_advance()
909 		BUG_ON(!cursor->resid);  in ceph_msg_data_bio_advance()
912 		if (cursor->resid <= bio_iter_len(bio, cursor->bvec_iter))  in ceph_msg_data_bio_advance()
913 			cursor->last_piece = true;  in ceph_msg_data_bio_advance()
924 static void ceph_msg_data_pages_cursor_init(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_pages_cursor_init()  argument
927 	struct ceph_msg_data *data = cursor->data;  in ceph_msg_data_pages_cursor_init()
935 	cursor->resid = min(length, data->length);  in ceph_msg_data_pages_cursor_init()
937 	cursor->page_offset = data->alignment & ~PAGE_MASK;  in ceph_msg_data_pages_cursor_init()
938 	cursor->page_index = 0;  in ceph_msg_data_pages_cursor_init()
940 	cursor->page_count = (unsigned short)page_count;  in ceph_msg_data_pages_cursor_init()
941 	BUG_ON(length > SIZE_MAX - cursor->page_offset);  in ceph_msg_data_pages_cursor_init()
942 	cursor->last_piece = cursor->page_offset + cursor->resid <= PAGE_SIZE;  in ceph_msg_data_pages_cursor_init()
946 ceph_msg_data_pages_next(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_pages_next()  argument
949 	struct ceph_msg_data *data = cursor->data;  in ceph_msg_data_pages_next()
953 	BUG_ON(cursor->page_index >= cursor->page_count);  in ceph_msg_data_pages_next()
954 	BUG_ON(cursor->page_offset >= PAGE_SIZE);  in ceph_msg_data_pages_next()
956 	*page_offset = cursor->page_offset;  in ceph_msg_data_pages_next()
957 	if (cursor->last_piece)  in ceph_msg_data_pages_next()
958 		*length = cursor->resid;  in ceph_msg_data_pages_next()
962 	return data->pages[cursor->page_index];  in ceph_msg_data_pages_next()
965 static bool ceph_msg_data_pages_advance(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_pages_advance()  argument
968 	BUG_ON(cursor->data->type != CEPH_MSG_DATA_PAGES);  in ceph_msg_data_pages_advance()
970 	BUG_ON(cursor->page_offset + bytes > PAGE_SIZE);  in ceph_msg_data_pages_advance()
974 	cursor->resid -= bytes;  in ceph_msg_data_pages_advance()
975 	cursor->page_offset = (cursor->page_offset + bytes) & ~PAGE_MASK;  in ceph_msg_data_pages_advance()
976 	if (!bytes || cursor->page_offset)  in ceph_msg_data_pages_advance()
979 	if (!cursor->resid)  in ceph_msg_data_pages_advance()
984 	BUG_ON(cursor->page_index >= cursor->page_count);  in ceph_msg_data_pages_advance()
985 	cursor->page_index++;  in ceph_msg_data_pages_advance()
986 	cursor->last_piece = cursor->resid <= PAGE_SIZE;  in ceph_msg_data_pages_advance()
996 ceph_msg_data_pagelist_cursor_init(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_pagelist_cursor_init()  argument
999 	struct ceph_msg_data *data = cursor->data;  in ceph_msg_data_pagelist_cursor_init()
1014 	cursor->resid = min(length, pagelist->length);  in ceph_msg_data_pagelist_cursor_init()
1015 	cursor->page = page;  in ceph_msg_data_pagelist_cursor_init()
1016 	cursor->offset = 0;  in ceph_msg_data_pagelist_cursor_init()
1017 	cursor->last_piece = cursor->resid <= PAGE_SIZE;  in ceph_msg_data_pagelist_cursor_init()
1021 ceph_msg_data_pagelist_next(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_pagelist_next()  argument
1024 	struct ceph_msg_data *data = cursor->data;  in ceph_msg_data_pagelist_next()
1032 	BUG_ON(!cursor->page);  in ceph_msg_data_pagelist_next()
1033 	BUG_ON(cursor->offset + cursor->resid != pagelist->length);  in ceph_msg_data_pagelist_next()
1036 	*page_offset = cursor->offset & ~PAGE_MASK;  in ceph_msg_data_pagelist_next()
1037 	if (cursor->last_piece)  in ceph_msg_data_pagelist_next()
1038 		*length = cursor->resid;  in ceph_msg_data_pagelist_next()
1042 	return cursor->page;  in ceph_msg_data_pagelist_next()
1045 static bool ceph_msg_data_pagelist_advance(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_pagelist_advance()  argument
1048 	struct ceph_msg_data *data = cursor->data;  in ceph_msg_data_pagelist_advance()
1056 	BUG_ON(cursor->offset + cursor->resid != pagelist->length);  in ceph_msg_data_pagelist_advance()
1057 	BUG_ON((cursor->offset & ~PAGE_MASK) + bytes > PAGE_SIZE);  in ceph_msg_data_pagelist_advance()
1061 	cursor->resid -= bytes;  in ceph_msg_data_pagelist_advance()
1062 	cursor->offset += bytes;  in ceph_msg_data_pagelist_advance()
1064 	if (!bytes || cursor->offset & ~PAGE_MASK)  in ceph_msg_data_pagelist_advance()
1067 	if (!cursor->resid)  in ceph_msg_data_pagelist_advance()
1072 	BUG_ON(list_is_last(&cursor->page->lru, &pagelist->head));  in ceph_msg_data_pagelist_advance()
1073 	cursor->page = list_entry_next(cursor->page, lru);  in ceph_msg_data_pagelist_advance()
1074 	cursor->last_piece = cursor->resid <= PAGE_SIZE;  in ceph_msg_data_pagelist_advance()
1087 static void __ceph_msg_data_cursor_init(struct ceph_msg_data_cursor *cursor)  in __ceph_msg_data_cursor_init()  argument
1089 	size_t length = cursor->total_resid;  in __ceph_msg_data_cursor_init()
1091 	switch (cursor->data->type) {  in __ceph_msg_data_cursor_init()
1093 		ceph_msg_data_pagelist_cursor_init(cursor, length);  in __ceph_msg_data_cursor_init()
1096 		ceph_msg_data_pages_cursor_init(cursor, length);  in __ceph_msg_data_cursor_init()
1100 		ceph_msg_data_bio_cursor_init(cursor, length);  in __ceph_msg_data_cursor_init()
1108 	cursor->need_crc = true;  in __ceph_msg_data_cursor_init()
1113 	struct ceph_msg_data_cursor *cursor = &msg->cursor;  in ceph_msg_data_cursor_init()  local
1120 	cursor->data_head = &msg->data;  in ceph_msg_data_cursor_init()
1121 	cursor->total_resid = length;  in ceph_msg_data_cursor_init()
1123 	cursor->data = data;  in ceph_msg_data_cursor_init()
1125 	__ceph_msg_data_cursor_init(cursor);  in ceph_msg_data_cursor_init()
1133 static struct page *ceph_msg_data_next(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_next()  argument
1139 	switch (cursor->data->type) {  in ceph_msg_data_next()
1141 		page = ceph_msg_data_pagelist_next(cursor, page_offset, length);  in ceph_msg_data_next()
1144 		page = ceph_msg_data_pages_next(cursor, page_offset, length);  in ceph_msg_data_next()
1148 		page = ceph_msg_data_bio_next(cursor, page_offset, length);  in ceph_msg_data_next()
1160 		*last_piece = cursor->last_piece;  in ceph_msg_data_next()
1169 static bool ceph_msg_data_advance(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_advance()  argument
1174 	BUG_ON(bytes > cursor->resid);  in ceph_msg_data_advance()
1175 	switch (cursor->data->type) {  in ceph_msg_data_advance()
1177 		new_piece = ceph_msg_data_pagelist_advance(cursor, bytes);  in ceph_msg_data_advance()
1180 		new_piece = ceph_msg_data_pages_advance(cursor, bytes);  in ceph_msg_data_advance()
1184 		new_piece = ceph_msg_data_bio_advance(cursor, bytes);  in ceph_msg_data_advance()
1192 	cursor->total_resid -= bytes;  in ceph_msg_data_advance()
1194 	if (!cursor->resid && cursor->total_resid) {  in ceph_msg_data_advance()
1195 		WARN_ON(!cursor->last_piece);  in ceph_msg_data_advance()
1196 		BUG_ON(list_is_last(&cursor->data->links, cursor->data_head));  in ceph_msg_data_advance()
1197 		cursor->data = list_entry_next(cursor->data, links);  in ceph_msg_data_advance()
1198 		__ceph_msg_data_cursor_init(cursor);  in ceph_msg_data_advance()
1201 	cursor->need_crc = new_piece;  in ceph_msg_data_advance()
1550 	struct ceph_msg_data_cursor *cursor = &msg->cursor;  in write_partial_message_data()  local
1568 	while (cursor->resid) {  in write_partial_message_data()
1576 		page = ceph_msg_data_next(&msg->cursor, &page_offset, &length,  in write_partial_message_data()
1586 		if (do_datacrc && cursor->need_crc)  in write_partial_message_data()
1588 		need_crc = ceph_msg_data_advance(&msg->cursor, (size_t)ret);  in write_partial_message_data()
2250 	struct ceph_msg_data_cursor *cursor = &msg->cursor;  in read_partial_msg_data()  local
2264 	while (cursor->resid) {  in read_partial_msg_data()
2265 		page = ceph_msg_data_next(&msg->cursor, &page_offset, &length,  in read_partial_msg_data()
2277 		(void) ceph_msg_data_advance(&msg->cursor, (size_t)ret);  in read_partial_msg_data()
3074 			con->out_skip += msg->cursor.total_resid;  in ceph_msg_revoke()