825 static void ceph_msg_data_bio_cursor_init(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_bio_cursor_init()  argument
828 	struct ceph_msg_data *data = cursor->data;  in ceph_msg_data_bio_cursor_init()
836 	cursor->resid = min(length, data->bio_length);  in ceph_msg_data_bio_cursor_init()
837 	cursor->bio = bio;  in ceph_msg_data_bio_cursor_init()
838 	cursor->bvec_iter = bio->bi_iter;  in ceph_msg_data_bio_cursor_init()
839 	cursor->last_piece =  in ceph_msg_data_bio_cursor_init()
840 		cursor->resid <= bio_iter_len(bio, cursor->bvec_iter);  in ceph_msg_data_bio_cursor_init()
843 static struct page *ceph_msg_data_bio_next(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_bio_next()  argument
847 	struct ceph_msg_data *data = cursor->data;  in ceph_msg_data_bio_next()
853 	bio = cursor->bio;  in ceph_msg_data_bio_next()
856 	bio_vec = bio_iter_iovec(bio, cursor->bvec_iter);  in ceph_msg_data_bio_next()
860 	if (cursor->last_piece) /* pagelist offset is always 0 */  in ceph_msg_data_bio_next()
861 		*length = cursor->resid;  in ceph_msg_data_bio_next()
864 	BUG_ON(*length > cursor->resid);  in ceph_msg_data_bio_next()
870 static bool ceph_msg_data_bio_advance(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_bio_advance()  argument
876 	BUG_ON(cursor->data->type != CEPH_MSG_DATA_BIO);  in ceph_msg_data_bio_advance()
878 	bio = cursor->bio;  in ceph_msg_data_bio_advance()
881 	bio_vec = bio_iter_iovec(bio, cursor->bvec_iter);  in ceph_msg_data_bio_advance()
885 	BUG_ON(cursor->resid < bytes);  in ceph_msg_data_bio_advance()
886 	cursor->resid -= bytes;  in ceph_msg_data_bio_advance()
888 	bio_advance_iter(bio, &cursor->bvec_iter, bytes);  in ceph_msg_data_bio_advance()
895 	if (!cursor->bvec_iter.bi_size) {  in ceph_msg_data_bio_advance()
897 		cursor->bio = bio;  in ceph_msg_data_bio_advance()
899 			cursor->bvec_iter = bio->bi_iter;  in ceph_msg_data_bio_advance()
901 			memset(&cursor->bvec_iter, 0,  in ceph_msg_data_bio_advance()
902 			       sizeof(cursor->bvec_iter));  in ceph_msg_data_bio_advance()
905 	if (!cursor->last_piece) {  in ceph_msg_data_bio_advance()
906 		BUG_ON(!cursor->resid);  in ceph_msg_data_bio_advance()
909 		if (cursor->resid <= bio_iter_len(bio, cursor->bvec_iter))  in ceph_msg_data_bio_advance()
910 			cursor->last_piece = true;  in ceph_msg_data_bio_advance()
921 static void ceph_msg_data_pages_cursor_init(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_pages_cursor_init()  argument
924 	struct ceph_msg_data *data = cursor->data;  in ceph_msg_data_pages_cursor_init()
932 	cursor->resid = min(length, data->length);  in ceph_msg_data_pages_cursor_init()
934 	cursor->page_offset = data->alignment & ~PAGE_MASK;  in ceph_msg_data_pages_cursor_init()
935 	cursor->page_index = 0;  in ceph_msg_data_pages_cursor_init()
937 	cursor->page_count = (unsigned short)page_count;  in ceph_msg_data_pages_cursor_init()
938 	BUG_ON(length > SIZE_MAX - cursor->page_offset);  in ceph_msg_data_pages_cursor_init()
939 	cursor->last_piece = cursor->page_offset + cursor->resid <= PAGE_SIZE;  in ceph_msg_data_pages_cursor_init()
943 ceph_msg_data_pages_next(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_pages_next()  argument
946 	struct ceph_msg_data *data = cursor->data;  in ceph_msg_data_pages_next()
950 	BUG_ON(cursor->page_index >= cursor->page_count);  in ceph_msg_data_pages_next()
951 	BUG_ON(cursor->page_offset >= PAGE_SIZE);  in ceph_msg_data_pages_next()
953 	*page_offset = cursor->page_offset;  in ceph_msg_data_pages_next()
954 	if (cursor->last_piece)  in ceph_msg_data_pages_next()
955 		*length = cursor->resid;  in ceph_msg_data_pages_next()
959 	return data->pages[cursor->page_index];  in ceph_msg_data_pages_next()
962 static bool ceph_msg_data_pages_advance(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_pages_advance()  argument
965 	BUG_ON(cursor->data->type != CEPH_MSG_DATA_PAGES);  in ceph_msg_data_pages_advance()
967 	BUG_ON(cursor->page_offset + bytes > PAGE_SIZE);  in ceph_msg_data_pages_advance()
971 	cursor->resid -= bytes;  in ceph_msg_data_pages_advance()
972 	cursor->page_offset = (cursor->page_offset + bytes) & ~PAGE_MASK;  in ceph_msg_data_pages_advance()
973 	if (!bytes || cursor->page_offset)  in ceph_msg_data_pages_advance()
976 	if (!cursor->resid)  in ceph_msg_data_pages_advance()
981 	BUG_ON(cursor->page_index >= cursor->page_count);  in ceph_msg_data_pages_advance()
982 	cursor->page_index++;  in ceph_msg_data_pages_advance()
983 	cursor->last_piece = cursor->resid <= PAGE_SIZE;  in ceph_msg_data_pages_advance()
993 ceph_msg_data_pagelist_cursor_init(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_pagelist_cursor_init()  argument
996 	struct ceph_msg_data *data = cursor->data;  in ceph_msg_data_pagelist_cursor_init()
1011 	cursor->resid = min(length, pagelist->length);  in ceph_msg_data_pagelist_cursor_init()
1012 	cursor->page = page;  in ceph_msg_data_pagelist_cursor_init()
1013 	cursor->offset = 0;  in ceph_msg_data_pagelist_cursor_init()
1014 	cursor->last_piece = cursor->resid <= PAGE_SIZE;  in ceph_msg_data_pagelist_cursor_init()
1018 ceph_msg_data_pagelist_next(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_pagelist_next()  argument
1021 	struct ceph_msg_data *data = cursor->data;  in ceph_msg_data_pagelist_next()
1029 	BUG_ON(!cursor->page);  in ceph_msg_data_pagelist_next()
1030 	BUG_ON(cursor->offset + cursor->resid != pagelist->length);  in ceph_msg_data_pagelist_next()
1033 	*page_offset = cursor->offset & ~PAGE_MASK;  in ceph_msg_data_pagelist_next()
1034 	if (cursor->last_piece)  in ceph_msg_data_pagelist_next()
1035 		*length = cursor->resid;  in ceph_msg_data_pagelist_next()
1039 	return cursor->page;  in ceph_msg_data_pagelist_next()
1042 static bool ceph_msg_data_pagelist_advance(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_pagelist_advance()  argument
1045 	struct ceph_msg_data *data = cursor->data;  in ceph_msg_data_pagelist_advance()
1053 	BUG_ON(cursor->offset + cursor->resid != pagelist->length);  in ceph_msg_data_pagelist_advance()
1054 	BUG_ON((cursor->offset & ~PAGE_MASK) + bytes > PAGE_SIZE);  in ceph_msg_data_pagelist_advance()
1058 	cursor->resid -= bytes;  in ceph_msg_data_pagelist_advance()
1059 	cursor->offset += bytes;  in ceph_msg_data_pagelist_advance()
1061 	if (!bytes || cursor->offset & ~PAGE_MASK)  in ceph_msg_data_pagelist_advance()
1064 	if (!cursor->resid)  in ceph_msg_data_pagelist_advance()
1069 	BUG_ON(list_is_last(&cursor->page->lru, &pagelist->head));  in ceph_msg_data_pagelist_advance()
1070 	cursor->page = list_entry_next(cursor->page, lru);  in ceph_msg_data_pagelist_advance()
1071 	cursor->last_piece = cursor->resid <= PAGE_SIZE;  in ceph_msg_data_pagelist_advance()
1084 static void __ceph_msg_data_cursor_init(struct ceph_msg_data_cursor *cursor)  in __ceph_msg_data_cursor_init()  argument
1086 	size_t length = cursor->total_resid;  in __ceph_msg_data_cursor_init()
1088 	switch (cursor->data->type) {  in __ceph_msg_data_cursor_init()
1090 		ceph_msg_data_pagelist_cursor_init(cursor, length);  in __ceph_msg_data_cursor_init()
1093 		ceph_msg_data_pages_cursor_init(cursor, length);  in __ceph_msg_data_cursor_init()
1097 		ceph_msg_data_bio_cursor_init(cursor, length);  in __ceph_msg_data_cursor_init()
1105 	cursor->need_crc = true;  in __ceph_msg_data_cursor_init()
1110 	struct ceph_msg_data_cursor *cursor = &msg->cursor;  in ceph_msg_data_cursor_init()  local
1117 	cursor->data_head = &msg->data;  in ceph_msg_data_cursor_init()
1118 	cursor->total_resid = length;  in ceph_msg_data_cursor_init()
1120 	cursor->data = data;  in ceph_msg_data_cursor_init()
1122 	__ceph_msg_data_cursor_init(cursor);  in ceph_msg_data_cursor_init()
1130 static struct page *ceph_msg_data_next(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_next()  argument
1136 	switch (cursor->data->type) {  in ceph_msg_data_next()
1138 		page = ceph_msg_data_pagelist_next(cursor, page_offset, length);  in ceph_msg_data_next()
1141 		page = ceph_msg_data_pages_next(cursor, page_offset, length);  in ceph_msg_data_next()
1145 		page = ceph_msg_data_bio_next(cursor, page_offset, length);  in ceph_msg_data_next()
1157 		*last_piece = cursor->last_piece;  in ceph_msg_data_next()
1166 static bool ceph_msg_data_advance(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_advance()  argument
1171 	BUG_ON(bytes > cursor->resid);  in ceph_msg_data_advance()
1172 	switch (cursor->data->type) {  in ceph_msg_data_advance()
1174 		new_piece = ceph_msg_data_pagelist_advance(cursor, bytes);  in ceph_msg_data_advance()
1177 		new_piece = ceph_msg_data_pages_advance(cursor, bytes);  in ceph_msg_data_advance()
1181 		new_piece = ceph_msg_data_bio_advance(cursor, bytes);  in ceph_msg_data_advance()
1189 	cursor->total_resid -= bytes;  in ceph_msg_data_advance()
1191 	if (!cursor->resid && cursor->total_resid) {  in ceph_msg_data_advance()
1192 		WARN_ON(!cursor->last_piece);  in ceph_msg_data_advance()
1193 		BUG_ON(list_is_last(&cursor->data->links, cursor->data_head));  in ceph_msg_data_advance()
1194 		cursor->data = list_entry_next(cursor->data, links);  in ceph_msg_data_advance()
1195 		__ceph_msg_data_cursor_init(cursor);  in ceph_msg_data_advance()
1198 	cursor->need_crc = new_piece;  in ceph_msg_data_advance()
1557 	struct ceph_msg_data_cursor *cursor = &msg->cursor;  in write_partial_message_data()  local
1575 	while (cursor->resid) {  in write_partial_message_data()
1583 		page = ceph_msg_data_next(cursor, &page_offset, &length,  in write_partial_message_data()
1593 		if (do_datacrc && cursor->need_crc)  in write_partial_message_data()
1595 		need_crc = ceph_msg_data_advance(cursor, (size_t)ret);  in write_partial_message_data()
2263 	struct ceph_msg_data_cursor *cursor = &msg->cursor;  in read_partial_msg_data()  local
2277 	while (cursor->resid) {  in read_partial_msg_data()
2278 		page = ceph_msg_data_next(cursor, &page_offset, &length, NULL);  in read_partial_msg_data()
2289 		(void) ceph_msg_data_advance(cursor, (size_t)ret);  in read_partial_msg_data()
3098 			con->out_skip += msg->cursor.total_resid;  in ceph_msg_revoke()