71 				   struct iscsi_segment *segment);
97 iscsi_tcp_segment_init_sg(struct iscsi_segment *segment,  in iscsi_tcp_segment_init_sg()  argument
100 	segment->sg = sg;  in iscsi_tcp_segment_init_sg()
101 	segment->sg_offset = offset;  in iscsi_tcp_segment_init_sg()
102 	segment->size = min(sg->length - offset,  in iscsi_tcp_segment_init_sg()
103 			    segment->total_size - segment->total_copied);  in iscsi_tcp_segment_init_sg()
104 	segment->data = NULL;  in iscsi_tcp_segment_init_sg()
116 static void iscsi_tcp_segment_map(struct iscsi_segment *segment, int recv)  in iscsi_tcp_segment_map()  argument
120 	if (segment->data != NULL || !segment->sg)  in iscsi_tcp_segment_map()
123 	sg = segment->sg;  in iscsi_tcp_segment_map()
124 	BUG_ON(segment->sg_mapped);  in iscsi_tcp_segment_map()
137 		segment->atomic_mapped = true;  in iscsi_tcp_segment_map()
138 		segment->sg_mapped = kmap_atomic(sg_page(sg));  in iscsi_tcp_segment_map()
140 		segment->atomic_mapped = false;  in iscsi_tcp_segment_map()
142 		segment->sg_mapped = kmap(sg_page(sg));  in iscsi_tcp_segment_map()
145 	segment->data = segment->sg_mapped + sg->offset + segment->sg_offset;  in iscsi_tcp_segment_map()
148 void iscsi_tcp_segment_unmap(struct iscsi_segment *segment)  in iscsi_tcp_segment_unmap()  argument
150 	if (segment->sg_mapped) {  in iscsi_tcp_segment_unmap()
151 		if (segment->atomic_mapped)  in iscsi_tcp_segment_unmap()
152 			kunmap_atomic(segment->sg_mapped);  in iscsi_tcp_segment_unmap()
154 			kunmap(sg_page(segment->sg));  in iscsi_tcp_segment_unmap()
155 		segment->sg_mapped = NULL;  in iscsi_tcp_segment_unmap()
156 		segment->data = NULL;  in iscsi_tcp_segment_unmap()
165 iscsi_tcp_segment_splice_digest(struct iscsi_segment *segment, void *digest)  in iscsi_tcp_segment_splice_digest()  argument
167 	segment->data = digest;  in iscsi_tcp_segment_splice_digest()
168 	segment->digest_len = ISCSI_DIGEST_SIZE;  in iscsi_tcp_segment_splice_digest()
169 	segment->total_size += ISCSI_DIGEST_SIZE;  in iscsi_tcp_segment_splice_digest()
170 	segment->size = ISCSI_DIGEST_SIZE;  in iscsi_tcp_segment_splice_digest()
171 	segment->copied = 0;  in iscsi_tcp_segment_splice_digest()
172 	segment->sg = NULL;  in iscsi_tcp_segment_splice_digest()
173 	segment->hash = NULL;  in iscsi_tcp_segment_splice_digest()
193 			   struct iscsi_segment *segment, int recv,  in iscsi_tcp_segment_done()  argument
200 		      segment->copied, copied, segment->size,  in iscsi_tcp_segment_done()
202 	if (segment->hash && copied) {  in iscsi_tcp_segment_done()
207 		iscsi_tcp_segment_unmap(segment);  in iscsi_tcp_segment_done()
209 		if (!segment->data) {  in iscsi_tcp_segment_done()
211 			sg_set_page(&sg, sg_page(segment->sg), copied,  in iscsi_tcp_segment_done()
212 				    segment->copied + segment->sg_offset +  in iscsi_tcp_segment_done()
213 							segment->sg->offset);  in iscsi_tcp_segment_done()
215 			sg_init_one(&sg, segment->data + segment->copied,  in iscsi_tcp_segment_done()
217 		crypto_hash_update(segment->hash, &sg, copied);  in iscsi_tcp_segment_done()
220 	segment->copied += copied;  in iscsi_tcp_segment_done()
221 	if (segment->copied < segment->size) {  in iscsi_tcp_segment_done()
222 		iscsi_tcp_segment_map(segment, recv);  in iscsi_tcp_segment_done()
226 	segment->total_copied += segment->copied;  in iscsi_tcp_segment_done()
227 	segment->copied = 0;  in iscsi_tcp_segment_done()
228 	segment->size = 0;  in iscsi_tcp_segment_done()
231 	iscsi_tcp_segment_unmap(segment);  in iscsi_tcp_segment_done()
235 		      segment->total_copied, segment->total_size);  in iscsi_tcp_segment_done()
236 	if (segment->total_copied < segment->total_size) {  in iscsi_tcp_segment_done()
238 		iscsi_tcp_segment_init_sg(segment, sg_next(segment->sg),  in iscsi_tcp_segment_done()
240 		iscsi_tcp_segment_map(segment, recv);  in iscsi_tcp_segment_done()
241 		BUG_ON(segment->size == 0);  in iscsi_tcp_segment_done()
247 		pad = iscsi_padding(segment->total_copied);  in iscsi_tcp_segment_done()
251 			segment->total_size += pad;  in iscsi_tcp_segment_done()
252 			segment->size = pad;  in iscsi_tcp_segment_done()
253 			segment->data = segment->padbuf;  in iscsi_tcp_segment_done()
262 	if (segment->hash) {  in iscsi_tcp_segment_done()
263 		crypto_hash_final(segment->hash, segment->digest);  in iscsi_tcp_segment_done()
264 		iscsi_tcp_segment_splice_digest(segment,  in iscsi_tcp_segment_done()
265 				 recv ? segment->recv_digest : segment->digest);  in iscsi_tcp_segment_done()
292 		       struct iscsi_segment *segment, const void *ptr,  in iscsi_tcp_segment_recv()  argument
297 	while (!iscsi_tcp_segment_done(tcp_conn, segment, 1, copy)) {  in iscsi_tcp_segment_recv()
304 		copy = min(len - copied, segment->size - segment->copied);  in iscsi_tcp_segment_recv()
306 		memcpy(segment->data + segment->copied, ptr + copied, copy);  in iscsi_tcp_segment_recv()
325 		      struct iscsi_segment *segment)  in iscsi_tcp_dgst_verify()  argument
327 	if (!segment->digest_len)  in iscsi_tcp_dgst_verify()
330 	if (memcmp(segment->recv_digest, segment->digest,  in iscsi_tcp_dgst_verify()
331 		   segment->digest_len)) {  in iscsi_tcp_dgst_verify()
343 __iscsi_segment_init(struct iscsi_segment *segment, size_t size,  in __iscsi_segment_init()  argument
346 	memset(segment, 0, sizeof(*segment));  in __iscsi_segment_init()
347 	segment->total_size = size;  in __iscsi_segment_init()
348 	segment->done = done;  in __iscsi_segment_init()
351 		segment->hash = hash;  in __iscsi_segment_init()
357 iscsi_segment_init_linear(struct iscsi_segment *segment, void *data,  in iscsi_segment_init_linear()  argument
361 	__iscsi_segment_init(segment, size, done, hash);  in iscsi_segment_init_linear()
362 	segment->data = data;  in iscsi_segment_init_linear()
363 	segment->size = size;  in iscsi_segment_init_linear()
368 iscsi_segment_seek_sg(struct iscsi_segment *segment,  in iscsi_segment_seek_sg()  argument
376 	__iscsi_segment_init(segment, size, done, hash);  in iscsi_segment_seek_sg()
379 			iscsi_tcp_segment_init_sg(segment, sg, offset);  in iscsi_segment_seek_sg()
402 	iscsi_segment_init_linear(&tcp_conn->in.segment,  in iscsi_tcp_hdr_recv_prep()
413 			 struct iscsi_segment *segment)  in iscsi_tcp_data_recv_done()  argument
418 	if (!iscsi_tcp_dgst_verify(tcp_conn, segment))  in iscsi_tcp_data_recv_done()
440 	iscsi_segment_init_linear(&tcp_conn->in.segment,  in iscsi_tcp_data_recv_prep()
614 			  struct iscsi_segment *segment)  in iscsi_tcp_process_data_in()  argument
620 	if (!iscsi_tcp_dgst_verify(tcp_conn, segment))  in iscsi_tcp_process_data_in()
709 			rc = iscsi_segment_seek_sg(&tcp_conn->in.segment,  in iscsi_tcp_hdr_dissect()
803 			struct iscsi_segment *segment)  in iscsi_tcp_hdr_recv_done()  argument
813 	if (segment->copied == sizeof(struct iscsi_hdr) && hdr->hlength) {  in iscsi_tcp_hdr_recv_done()
823 		segment->total_size += ahslen;  in iscsi_tcp_hdr_recv_done()
824 		segment->size += ahslen;  in iscsi_tcp_hdr_recv_done()
833 		if (segment->digest_len == 0) {  in iscsi_tcp_hdr_recv_done()
839 			iscsi_tcp_segment_splice_digest(segment,  in iscsi_tcp_hdr_recv_done()
840 							segment->recv_digest);  in iscsi_tcp_hdr_recv_done()
845 				      segment->total_copied - ISCSI_DIGEST_SIZE,  in iscsi_tcp_hdr_recv_done()
846 				      segment->digest);  in iscsi_tcp_hdr_recv_done()
848 		if (!iscsi_tcp_dgst_verify(tcp_conn, segment))  in iscsi_tcp_hdr_recv_done()
865 	return tcp_conn->in.segment.done == iscsi_tcp_hdr_recv_done;  in iscsi_tcp_recv_segment_is_hdr()
883 	struct iscsi_segment *segment = &tcp_conn->in.segment;  in iscsi_tcp_recv_skb()  local
903 		segment->total_copied = segment->total_size;  in iscsi_tcp_recv_skb()
919 		BUG_ON(segment->copied >= segment->size);  in iscsi_tcp_recv_skb()
923 		rc = iscsi_tcp_segment_recv(tcp_conn, segment, ptr, avail);  in iscsi_tcp_recv_skb()
927 		if (segment->total_copied >= segment->total_size) {  in iscsi_tcp_recv_skb()
936 	rc = segment->done(tcp_conn, segment);  in iscsi_tcp_recv_skb()