267 		struct ehca_mr_pginfo pginfo;  in ehca_reg_phys_mr()  local
278 		memset(&pginfo, 0, sizeof(pginfo));  in ehca_reg_phys_mr()
279 		pginfo.type = EHCA_MR_PGI_PHYS;  in ehca_reg_phys_mr()
280 		pginfo.num_kpages = num_kpages;  in ehca_reg_phys_mr()
281 		pginfo.hwpage_size = hw_pgsize;  in ehca_reg_phys_mr()
282 		pginfo.num_hwpages = num_hwpages;  in ehca_reg_phys_mr()
283 		pginfo.u.phy.num_phys_buf = num_phys_buf;  in ehca_reg_phys_mr()
284 		pginfo.u.phy.phys_buf_array = phys_buf_array;  in ehca_reg_phys_mr()
285 		pginfo.next_hwpage =  in ehca_reg_phys_mr()
289 				  e_pd, &pginfo, &e_mr->ib.ib_mr.lkey,  in ehca_reg_phys_mr()
322 	struct ehca_mr_pginfo pginfo;  in ehca_reg_user_mr()  local
396 	memset(&pginfo, 0, sizeof(pginfo));  in ehca_reg_user_mr()
397 	pginfo.type = EHCA_MR_PGI_USER;  in ehca_reg_user_mr()
398 	pginfo.hwpage_size = hwpage_size;  in ehca_reg_user_mr()
399 	pginfo.num_kpages = num_kpages;  in ehca_reg_user_mr()
400 	pginfo.num_hwpages = num_hwpages;  in ehca_reg_user_mr()
401 	pginfo.u.usr.region = e_mr->umem;  in ehca_reg_user_mr()
402 	pginfo.next_hwpage = ib_umem_offset(e_mr->umem) / hwpage_size;  in ehca_reg_user_mr()
403 	pginfo.u.usr.next_sg = pginfo.u.usr.region->sg_head.sgl;  in ehca_reg_user_mr()
405 			  e_pd, &pginfo, &e_mr->ib.ib_mr.lkey,  in ehca_reg_user_mr()
407 	if (ret == -EINVAL && pginfo.hwpage_size > PAGE_SIZE) {  in ehca_reg_user_mr()
461 	struct ehca_mr_pginfo pginfo;  in ehca_rereg_phys_mr()  local
554 		memset(&pginfo, 0, sizeof(pginfo));  in ehca_rereg_phys_mr()
555 		pginfo.type = EHCA_MR_PGI_PHYS;  in ehca_rereg_phys_mr()
556 		pginfo.num_kpages = num_kpages;  in ehca_rereg_phys_mr()
557 		pginfo.hwpage_size = hw_pgsize;  in ehca_rereg_phys_mr()
558 		pginfo.num_hwpages = num_hwpages;  in ehca_rereg_phys_mr()
559 		pginfo.u.phy.num_phys_buf = num_phys_buf;  in ehca_rereg_phys_mr()
560 		pginfo.u.phy.phys_buf_array = phys_buf_array;  in ehca_rereg_phys_mr()
561 		pginfo.next_hwpage =  in ehca_rereg_phys_mr()
570 			    new_pd, &pginfo, &tmp_lkey, &tmp_rkey);  in ehca_rereg_phys_mr()
776 	struct ehca_mr_pginfo pginfo;  in ehca_alloc_fmr()  local
824 	memset(&pginfo, 0, sizeof(pginfo));  in ehca_alloc_fmr()
825 	pginfo.hwpage_size = hw_pgsize;  in ehca_alloc_fmr()
832 			  mr_access_flags, e_pd, &pginfo,  in ehca_alloc_fmr()
865 	struct ehca_mr_pginfo pginfo;  in ehca_map_phys_fmr()  local
891 	memset(&pginfo, 0, sizeof(pginfo));  in ehca_map_phys_fmr()
892 	pginfo.type = EHCA_MR_PGI_FMR;  in ehca_map_phys_fmr()
893 	pginfo.num_kpages = list_len;  in ehca_map_phys_fmr()
894 	pginfo.hwpage_size = e_fmr->hwpage_size;  in ehca_map_phys_fmr()
895 	pginfo.num_hwpages =  in ehca_map_phys_fmr()
896 		list_len * e_fmr->fmr_page_size / pginfo.hwpage_size;  in ehca_map_phys_fmr()
897 	pginfo.u.fmr.page_list = page_list;  in ehca_map_phys_fmr()
898 	pginfo.next_hwpage =  in ehca_map_phys_fmr()
899 		(iova & (e_fmr->fmr_page_size-1)) / pginfo.hwpage_size;  in ehca_map_phys_fmr()
900 	pginfo.u.fmr.fmr_pgsize = e_fmr->fmr_page_size;  in ehca_map_phys_fmr()
904 			    e_fmr->acl, e_pd, &pginfo, &tmp_lkey, &tmp_rkey);  in ehca_map_phys_fmr()
1019 				   struct ehca_mr_pginfo *pginfo);
1027 		struct ehca_mr_pginfo *pginfo,  in ehca_reg_mr()  argument
1038 	ehca_mrmw_set_pgsize_hipz_acl(pginfo->hwpage_size, &hipz_acl);  in ehca_reg_mr()
1055 		ret = ehca_reg_bmap_mr_rpages(shca, e_mr, pginfo);  in ehca_reg_mr()
1057 		ret = ehca_reg_mr_rpages(shca, e_mr, pginfo);  in ehca_reg_mr()
1065 	e_mr->num_kpages = pginfo->num_kpages;  in ehca_reg_mr()
1066 	e_mr->num_hwpages = pginfo->num_hwpages;  in ehca_reg_mr()
1067 	e_mr->hwpage_size = pginfo->hwpage_size;  in ehca_reg_mr()
1082 			 hipzout.lkey, pginfo, pginfo->num_kpages,  in ehca_reg_mr()
1083 			 pginfo->num_hwpages, ret);  in ehca_reg_mr()
1092 			 ret, shca, e_mr, iova_start, size, acl, e_pd, pginfo,  in ehca_reg_mr()
1093 			 pginfo->num_kpages, pginfo->num_hwpages);  in ehca_reg_mr()
1101 		       struct ehca_mr_pginfo *pginfo)  in ehca_reg_mr_rpages()  argument
1110 	if (!pginfo->num_hwpages) /* in case of fmr */  in ehca_reg_mr_rpages()
1121 	for (i = 0; i < NUM_CHUNKS(pginfo->num_hwpages, MAX_RPAGES); i++) {  in ehca_reg_mr_rpages()
1123 		if (i == NUM_CHUNKS(pginfo->num_hwpages, MAX_RPAGES) - 1) {  in ehca_reg_mr_rpages()
1124 			rnum = pginfo->num_hwpages % MAX_RPAGES; /* last shot */  in ehca_reg_mr_rpages()
1130 		ret = ehca_set_pagebuf(pginfo, rnum, kpage);  in ehca_reg_mr_rpages()
1151 			ehca_encode_hwpage_size(pginfo->hwpage_size),  in ehca_reg_mr_rpages()
1154 		if (i == NUM_CHUNKS(pginfo->num_hwpages, MAX_RPAGES) - 1) {  in ehca_reg_mr_rpages()
1191 			 pginfo, pginfo->num_kpages, pginfo->num_hwpages);  in ehca_reg_mr_rpages()
1203 				struct ehca_mr_pginfo *pginfo,  in ehca_rereg_mr_rereg1()  argument
1216 	ehca_mrmw_set_pgsize_hipz_acl(pginfo->hwpage_size, &hipz_acl);  in ehca_rereg_mr_rereg1()
1225 	pginfo_save = *pginfo;  in ehca_rereg_mr_rereg1()
1226 	ret = ehca_set_pagebuf(pginfo, pginfo->num_hwpages, kpage);  in ehca_rereg_mr_rereg1()
1230 			 "kpage=%p", e_mr, pginfo, pginfo->type,  in ehca_rereg_mr_rereg1()
1231 			 pginfo->num_kpages, pginfo->num_hwpages, kpage);  in ehca_rereg_mr_rereg1()
1251 		*pginfo = pginfo_save;  in ehca_rereg_mr_rereg1()
1265 		e_mr->num_kpages = pginfo->num_kpages;  in ehca_rereg_mr_rereg1()
1266 		e_mr->num_hwpages = pginfo->num_hwpages;  in ehca_rereg_mr_rereg1()
1267 		e_mr->hwpage_size = pginfo->hwpage_size;  in ehca_rereg_mr_rereg1()
1281 			 ret, *lkey, *rkey, pginfo, pginfo->num_kpages,  in ehca_rereg_mr_rereg1()
1282 			 pginfo->num_hwpages);  in ehca_rereg_mr_rereg1()
1294 		  struct ehca_mr_pginfo *pginfo,  in ehca_rereg_mr()  argument
1304 	if ((pginfo->num_hwpages > MAX_RPAGES) ||  in ehca_rereg_mr()
1306 	    (pginfo->num_hwpages > e_mr->num_hwpages)) {  in ehca_rereg_mr()
1309 			 pginfo->num_hwpages, e_mr->num_hwpages);  in ehca_rereg_mr()
1324 					   acl, e_pd, pginfo, lkey, rkey);  in ehca_rereg_mr()
1361 				  e_pd, pginfo, lkey, rkey, EHCA_REG_MR);  in ehca_rereg_mr()
1376 			 acl, e_pd, pginfo, pginfo->num_kpages, *lkey, *rkey,  in ehca_rereg_mr()
1392 	struct ehca_mr_pginfo pginfo;  in ehca_unmap_one_fmr()  local
1449 	memset(&pginfo, 0, sizeof(pginfo));  in ehca_unmap_one_fmr()
1450 	pginfo.type = EHCA_MR_PGI_FMR;  in ehca_unmap_one_fmr()
1453 			  e_fmr->acl, e_pd, &pginfo, &tmp_lkey,  in ehca_unmap_one_fmr()
1536 			       struct ehca_mr_pginfo *pginfo)  in ehca_reg_mr_section()  argument
1544 	if ((unsigned long)sectbase & (pginfo->hwpage_size - 1)) {  in ehca_reg_mr_section()
1549 	page_count = EHCA_SECTSIZE / pginfo->hwpage_size;  in ehca_reg_mr_section()
1555 			void *pg = sectbase + ((page++) * pginfo->hwpage_size);  in ehca_reg_mr_section()
1560 			ehca_encode_hwpage_size(pginfo->hwpage_size),  in ehca_reg_mr_section()
1573 				struct ehca_mr_pginfo *pginfo)  in ehca_reg_mr_sections()  argument
1583 					   pginfo);  in ehca_reg_mr_sections()
1592 				    struct ehca_mr_pginfo *pginfo)  in ehca_reg_mr_dir_sections()  argument
1601 		hret = ehca_reg_mr_sections(top, dir, kpage, shca, mr, pginfo);  in ehca_reg_mr_dir_sections()
1618 	struct ehca_mr_pginfo pginfo;  in ehca_reg_internal_maxmr()  local
1648 	memset(&pginfo, 0, sizeof(pginfo));  in ehca_reg_internal_maxmr()
1649 	pginfo.type = EHCA_MR_PGI_PHYS;  in ehca_reg_internal_maxmr()
1650 	pginfo.num_kpages = num_kpages;  in ehca_reg_internal_maxmr()
1651 	pginfo.num_hwpages = num_hwpages;  in ehca_reg_internal_maxmr()
1652 	pginfo.hwpage_size = hw_pgsize;  in ehca_reg_internal_maxmr()
1653 	pginfo.u.phy.num_phys_buf = 1;  in ehca_reg_internal_maxmr()
1654 	pginfo.u.phy.phys_buf_array = &ib_pbuf;  in ehca_reg_internal_maxmr()
1657 			  &pginfo, &e_mr->ib.ib_mr.lkey,  in ehca_reg_internal_maxmr()
1853 static int ehca_set_pagebuf_user1(struct ehca_mr_pginfo *pginfo,  in ehca_set_pagebuf_user1()  argument
1860 	int hwpages_per_kpage = PAGE_SIZE / pginfo->hwpage_size;  in ehca_set_pagebuf_user1()
1861 	struct scatterlist **sg = &pginfo->u.usr.next_sg;  in ehca_set_pagebuf_user1()
1866 		*kpage = pgaddr + (pginfo->next_hwpage *  in ehca_set_pagebuf_user1()
1867 				   pginfo->hwpage_size);  in ehca_set_pagebuf_user1()
1873 				     pginfo->u.usr.next_nmap,  in ehca_set_pagebuf_user1()
1874 				     pginfo->next_hwpage);  in ehca_set_pagebuf_user1()
1877 		(pginfo->hwpage_cnt)++;  in ehca_set_pagebuf_user1()
1878 		(pginfo->next_hwpage)++;  in ehca_set_pagebuf_user1()
1880 		if (pginfo->next_hwpage % hwpages_per_kpage == 0) {  in ehca_set_pagebuf_user1()
1881 			(pginfo->kpage_cnt)++;  in ehca_set_pagebuf_user1()
1882 			(pginfo->u.usr.next_nmap)++;  in ehca_set_pagebuf_user1()
1883 			pginfo->next_hwpage = 0;  in ehca_set_pagebuf_user1()
1919 static int ehca_set_pagebuf_user2(struct ehca_mr_pginfo *pginfo,  in ehca_set_pagebuf_user2()  argument
1926 	int kpages_per_hwpage = pginfo->hwpage_size / PAGE_SIZE;  in ehca_set_pagebuf_user2()
1928 	struct scatterlist **sg = &pginfo->u.usr.next_sg;  in ehca_set_pagebuf_user2()
1938 					     pgaddr, pginfo->u.usr.next_nmap);  in ehca_set_pagebuf_user2()
1946 			if (pgaddr & (pginfo->hwpage_size - 1)) {  in ehca_set_pagebuf_user2()
1947 				if (pginfo->hwpage_cnt) {  in ehca_set_pagebuf_user2()
1952 						pgaddr, pginfo->u.usr.next_nmap,  in ehca_set_pagebuf_user2()
1953 						pginfo->hwpage_size);  in ehca_set_pagebuf_user2()
1958 				pginfo->kpage_cnt =  in ehca_set_pagebuf_user2()
1960 					 (pginfo->hwpage_size - 1)) >>  in ehca_set_pagebuf_user2()
1962 				nr_kpages -= pginfo->kpage_cnt;  in ehca_set_pagebuf_user2()
1964 					 ~(pginfo->hwpage_size - 1);  in ehca_set_pagebuf_user2()
1974 			pginfo->kpage_cnt++;  in ehca_set_pagebuf_user2()
1975 			pginfo->u.usr.next_nmap++;  in ehca_set_pagebuf_user2()
1986 		pginfo->kpage_cnt += nr_kpages;  in ehca_set_pagebuf_user2()
1987 		pginfo->u.usr.next_nmap += nr_kpages;  in ehca_set_pagebuf_user2()
1991 		(pginfo->hwpage_cnt)++;  in ehca_set_pagebuf_user2()
2001 static int ehca_set_pagebuf_phys(struct ehca_mr_pginfo *pginfo,  in ehca_set_pagebuf_phys()  argument
2011 		pbuf   = pginfo->u.phy.phys_buf_array + pginfo->u.phy.next_buf;  in ehca_set_pagebuf_phys()
2012 		num_hw  = NUM_CHUNKS((pbuf->addr % pginfo->hwpage_size) +  in ehca_set_pagebuf_phys()
2013 				     pbuf->size, pginfo->hwpage_size);  in ehca_set_pagebuf_phys()
2014 		offs_hw = (pbuf->addr & ~(pginfo->hwpage_size - 1)) /  in ehca_set_pagebuf_phys()
2015 			pginfo->hwpage_size;  in ehca_set_pagebuf_phys()
2016 		while (pginfo->next_hwpage < offs_hw + num_hw) {  in ehca_set_pagebuf_phys()
2018 			if ((pginfo->kpage_cnt >= pginfo->num_kpages) ||  in ehca_set_pagebuf_phys()
2019 			    (pginfo->hwpage_cnt >= pginfo->num_hwpages)) {  in ehca_set_pagebuf_phys()
2024 					     pginfo->kpage_cnt,  in ehca_set_pagebuf_phys()
2025 					     pginfo->num_kpages,  in ehca_set_pagebuf_phys()
2026 					     pginfo->hwpage_cnt,  in ehca_set_pagebuf_phys()
2027 					     pginfo->num_hwpages, i);  in ehca_set_pagebuf_phys()
2030 			*kpage = (pbuf->addr & ~(pginfo->hwpage_size - 1)) +  in ehca_set_pagebuf_phys()
2031 				 (pginfo->next_hwpage * pginfo->hwpage_size);  in ehca_set_pagebuf_phys()
2035 					     pbuf->size, pginfo->next_hwpage);  in ehca_set_pagebuf_phys()
2038 			(pginfo->hwpage_cnt)++;  in ehca_set_pagebuf_phys()
2039 			(pginfo->next_hwpage)++;  in ehca_set_pagebuf_phys()
2040 			if (PAGE_SIZE >= pginfo->hwpage_size) {  in ehca_set_pagebuf_phys()
2041 				if (pginfo->next_hwpage %  in ehca_set_pagebuf_phys()
2042 				    (PAGE_SIZE / pginfo->hwpage_size) == 0)  in ehca_set_pagebuf_phys()
2043 					(pginfo->kpage_cnt)++;  in ehca_set_pagebuf_phys()
2045 				pginfo->kpage_cnt += pginfo->hwpage_size /  in ehca_set_pagebuf_phys()
2051 		if (pginfo->next_hwpage >= offs_hw + num_hw) {  in ehca_set_pagebuf_phys()
2052 			(pginfo->u.phy.next_buf)++;  in ehca_set_pagebuf_phys()
2053 			pginfo->next_hwpage = 0;  in ehca_set_pagebuf_phys()
2059 static int ehca_set_pagebuf_fmr(struct ehca_mr_pginfo *pginfo,  in ehca_set_pagebuf_fmr()  argument
2067 	fmrlist = pginfo->u.fmr.page_list + pginfo->u.fmr.next_listelem;  in ehca_set_pagebuf_fmr()
2069 		*kpage = (*fmrlist & ~(pginfo->hwpage_size - 1)) +  in ehca_set_pagebuf_fmr()
2070 			   pginfo->next_hwpage * pginfo->hwpage_size;  in ehca_set_pagebuf_fmr()
2075 				     pginfo->u.fmr.next_listelem,  in ehca_set_pagebuf_fmr()
2076 				     pginfo->next_hwpage);  in ehca_set_pagebuf_fmr()
2079 		(pginfo->hwpage_cnt)++;  in ehca_set_pagebuf_fmr()
2080 		if (pginfo->u.fmr.fmr_pgsize >= pginfo->hwpage_size) {  in ehca_set_pagebuf_fmr()
2081 			if (pginfo->next_hwpage %  in ehca_set_pagebuf_fmr()
2082 			    (pginfo->u.fmr.fmr_pgsize /  in ehca_set_pagebuf_fmr()
2083 			     pginfo->hwpage_size) == 0) {  in ehca_set_pagebuf_fmr()
2084 				(pginfo->kpage_cnt)++;  in ehca_set_pagebuf_fmr()
2085 				(pginfo->u.fmr.next_listelem)++;  in ehca_set_pagebuf_fmr()
2087 				pginfo->next_hwpage = 0;  in ehca_set_pagebuf_fmr()
2089 				(pginfo->next_hwpage)++;  in ehca_set_pagebuf_fmr()
2091 			unsigned int cnt_per_hwpage = pginfo->hwpage_size /  in ehca_set_pagebuf_fmr()
2092 				pginfo->u.fmr.fmr_pgsize;  in ehca_set_pagebuf_fmr()
2097 				u64 p = fmrlist[j] & ~(pginfo->hwpage_size - 1);  in ehca_set_pagebuf_fmr()
2098 				if (prev + pginfo->u.fmr.fmr_pgsize != p) {  in ehca_set_pagebuf_fmr()
2106 			pginfo->kpage_cnt += cnt_per_hwpage;  in ehca_set_pagebuf_fmr()
2107 			pginfo->u.fmr.next_listelem += cnt_per_hwpage;  in ehca_set_pagebuf_fmr()
2116 int ehca_set_pagebuf(struct ehca_mr_pginfo *pginfo,  in ehca_set_pagebuf()  argument
2122 	switch (pginfo->type) {  in ehca_set_pagebuf()
2124 		ret = ehca_set_pagebuf_phys(pginfo, number, kpage);  in ehca_set_pagebuf()
2127 		ret = PAGE_SIZE >= pginfo->hwpage_size ?  in ehca_set_pagebuf()
2128 			ehca_set_pagebuf_user1(pginfo, number, kpage) :  in ehca_set_pagebuf()
2129 			ehca_set_pagebuf_user2(pginfo, number, kpage);  in ehca_set_pagebuf()
2132 		ret = ehca_set_pagebuf_fmr(pginfo, number, kpage);  in ehca_set_pagebuf()
2135 		ehca_gen_err("bad pginfo->type=%x", pginfo->type);  in ehca_set_pagebuf()
2410 				   struct ehca_mr_pginfo *pginfo)  in ehca_reg_bmap_mr_rpages()  argument
2423 		hret = ehca_reg_mr_dir_sections(top, kpage, shca, e_mr, pginfo);  in ehca_reg_bmap_mr_rpages()