173 	struct ipath_sge *sge = &ss->sge;  in ipath_copy_sge()  local
176 		u32 len = sge->length;  in ipath_copy_sge()
180 		if (len > sge->sge_length)  in ipath_copy_sge()
181 			len = sge->sge_length;  in ipath_copy_sge()
183 		memcpy(sge->vaddr, data, len);  in ipath_copy_sge()
184 		sge->vaddr += len;  in ipath_copy_sge()
185 		sge->length -= len;  in ipath_copy_sge()
186 		sge->sge_length -= len;  in ipath_copy_sge()
187 		if (sge->sge_length == 0) {  in ipath_copy_sge()
189 				*sge = *ss->sg_list++;  in ipath_copy_sge()
190 		} else if (sge->length == 0 && sge->mr != NULL) {  in ipath_copy_sge()
191 			if (++sge->n >= IPATH_SEGSZ) {  in ipath_copy_sge()
192 				if (++sge->m >= sge->mr->mapsz)  in ipath_copy_sge()
194 				sge->n = 0;  in ipath_copy_sge()
196 			sge->vaddr =  in ipath_copy_sge()
197 				sge->mr->map[sge->m]->segs[sge->n].vaddr;  in ipath_copy_sge()
198 			sge->length =  in ipath_copy_sge()
199 				sge->mr->map[sge->m]->segs[sge->n].length;  in ipath_copy_sge()
213 	struct ipath_sge *sge = &ss->sge;  in ipath_skip_sge()  local
216 		u32 len = sge->length;  in ipath_skip_sge()
220 		if (len > sge->sge_length)  in ipath_skip_sge()
221 			len = sge->sge_length;  in ipath_skip_sge()
223 		sge->vaddr += len;  in ipath_skip_sge()
224 		sge->length -= len;  in ipath_skip_sge()
225 		sge->sge_length -= len;  in ipath_skip_sge()
226 		if (sge->sge_length == 0) {  in ipath_skip_sge()
228 				*sge = *ss->sg_list++;  in ipath_skip_sge()
229 		} else if (sge->length == 0 && sge->mr != NULL) {  in ipath_skip_sge()
230 			if (++sge->n >= IPATH_SEGSZ) {  in ipath_skip_sge()
231 				if (++sge->m >= sge->mr->mapsz)  in ipath_skip_sge()
233 				sge->n = 0;  in ipath_skip_sge()
235 			sge->vaddr =  in ipath_skip_sge()
236 				sge->mr->map[sge->m]->segs[sge->n].vaddr;  in ipath_skip_sge()
237 			sge->length =  in ipath_skip_sge()
238 				sge->mr->map[sge->m]->segs[sge->n].length;  in ipath_skip_sge()
252 	struct ipath_sge sge = ss->sge;  in ipath_count_sge()  local
257 		u32 len = sge.length;  in ipath_count_sge()
261 		if (len > sge.sge_length)  in ipath_count_sge()
262 			len = sge.sge_length;  in ipath_count_sge()
264 		if (((long) sge.vaddr & (sizeof(u32) - 1)) ||  in ipath_count_sge()
270 		sge.vaddr += len;  in ipath_count_sge()
271 		sge.length -= len;  in ipath_count_sge()
272 		sge.sge_length -= len;  in ipath_count_sge()
273 		if (sge.sge_length == 0) {  in ipath_count_sge()
275 				sge = *sg_list++;  in ipath_count_sge()
276 		} else if (sge.length == 0 && sge.mr != NULL) {  in ipath_count_sge()
277 			if (++sge.n >= IPATH_SEGSZ) {  in ipath_count_sge()
278 				if (++sge.m >= sge.mr->mapsz)  in ipath_count_sge()
280 				sge.n = 0;  in ipath_count_sge()
282 			sge.vaddr =  in ipath_count_sge()
283 				sge.mr->map[sge.m]->segs[sge.n].vaddr;  in ipath_count_sge()
284 			sge.length =  in ipath_count_sge()
285 				sge.mr->map[sge.m]->segs[sge.n].length;  in ipath_count_sge()
298 	struct ipath_sge *sge = &ss->sge;  in ipath_copy_from_sge()  local
301 		u32 len = sge->length;  in ipath_copy_from_sge()
305 		if (len > sge->sge_length)  in ipath_copy_from_sge()
306 			len = sge->sge_length;  in ipath_copy_from_sge()
308 		memcpy(data, sge->vaddr, len);  in ipath_copy_from_sge()
309 		sge->vaddr += len;  in ipath_copy_from_sge()
310 		sge->length -= len;  in ipath_copy_from_sge()
311 		sge->sge_length -= len;  in ipath_copy_from_sge()
312 		if (sge->sge_length == 0) {  in ipath_copy_from_sge()
314 				*sge = *ss->sg_list++;  in ipath_copy_from_sge()
315 		} else if (sge->length == 0 && sge->mr != NULL) {  in ipath_copy_from_sge()
316 			if (++sge->n >= IPATH_SEGSZ) {  in ipath_copy_from_sge()
317 				if (++sge->m >= sge->mr->mapsz)  in ipath_copy_from_sge()
319 				sge->n = 0;  in ipath_copy_from_sge()
321 			sge->vaddr =  in ipath_copy_from_sge()
322 				sge->mr->map[sge->m]->segs[sge->n].vaddr;  in ipath_copy_from_sge()
323 			sge->length =  in ipath_copy_from_sge()
324 				sge->mr->map[sge->m]->segs[sge->n].length;  in ipath_copy_from_sge()
782 	struct ipath_sge *sge = &ss->sge;  in update_sge()  local
784 	sge->vaddr += length;  in update_sge()
785 	sge->length -= length;  in update_sge()
786 	sge->sge_length -= length;  in update_sge()
787 	if (sge->sge_length == 0) {  in update_sge()
789 			*sge = *ss->sg_list++;  in update_sge()
790 	} else if (sge->length == 0 && sge->mr != NULL) {  in update_sge()
791 		if (++sge->n >= IPATH_SEGSZ) {  in update_sge()
792 			if (++sge->m >= sge->mr->mapsz)  in update_sge()
794 			sge->n = 0;  in update_sge()
796 		sge->vaddr = sge->mr->map[sge->m]->segs[sge->n].vaddr;  in update_sge()
797 		sge->length = sge->mr->map[sge->m]->segs[sge->n].length;  in update_sge()
845 		u32 len = ss->sge.length;  in copy_io()
850 		if (len > ss->sge.sge_length)  in copy_io()
851 			len = ss->sge.sge_length;  in copy_io()
854 		off = (unsigned long)ss->sge.vaddr & (sizeof(u32) - 1);  in copy_io()
856 			u32 *addr = (u32 *)((unsigned long)ss->sge.vaddr &  in copy_io()
887 			u32 *addr = (u32 *) ss->sge.vaddr;  in copy_io()
941 			__iowrite32_copy(piobuf, ss->sge.vaddr, w - 1);  in copy_io()
943 			last = ((u32 *) ss->sge.vaddr)[w - 1];  in copy_io()
948 			__iowrite32_copy(piobuf, ss->sge.vaddr, w);  in copy_io()
953 				u32 v = ((u32 *) ss->sge.vaddr)[w];  in copy_io()
1285 	if (likely(ss->num_sge == 1 && len <= ss->sge.length &&  in ipath_verbs_send_pio()
1286 		   !((unsigned long)ss->sge.vaddr & (sizeof(u32) - 1)))) {  in ipath_verbs_send_pio()
1287 		u32 *addr = (u32 *) ss->sge.vaddr;  in ipath_verbs_send_pio()