176 static void its_encode_cmd(struct its_cmd_block *cmd, u8 cmd_nr)
178 	cmd->raw_cmd[0] &= ~0xffUL;
179 	cmd->raw_cmd[0] |= cmd_nr;
182 static void its_encode_devid(struct its_cmd_block *cmd, u32 devid)  in its_encode_devid()  argument
184 	cmd->raw_cmd[0] &= BIT_ULL(32) - 1;  in its_encode_devid()
185 	cmd->raw_cmd[0] |= ((u64)devid) << 32;  in its_encode_devid()
188 static void its_encode_event_id(struct its_cmd_block *cmd, u32 id)  in its_encode_event_id()  argument
190 	cmd->raw_cmd[1] &= ~0xffffffffUL;  in its_encode_event_id()
191 	cmd->raw_cmd[1] |= id;  in its_encode_event_id()
194 static void its_encode_phys_id(struct its_cmd_block *cmd, u32 phys_id)  in its_encode_phys_id()  argument
196 	cmd->raw_cmd[1] &= 0xffffffffUL;  in its_encode_phys_id()
197 	cmd->raw_cmd[1] |= ((u64)phys_id) << 32;  in its_encode_phys_id()
200 static void its_encode_size(struct its_cmd_block *cmd, u8 size)  in its_encode_size()  argument
202 	cmd->raw_cmd[1] &= ~0x1fUL;  in its_encode_size()
203 	cmd->raw_cmd[1] |= size & 0x1f;  in its_encode_size()
206 static void its_encode_itt(struct its_cmd_block *cmd, u64 itt_addr)  in its_encode_itt()  argument
208 	cmd->raw_cmd[2] &= ~0xffffffffffffUL;  in its_encode_itt()
209 	cmd->raw_cmd[2] |= itt_addr & 0xffffffffff00UL;  in its_encode_itt()
212 static void its_encode_valid(struct its_cmd_block *cmd, int valid)  in its_encode_valid()  argument
214 	cmd->raw_cmd[2] &= ~(1UL << 63);  in its_encode_valid()
215 	cmd->raw_cmd[2] |= ((u64)!!valid) << 63;  in its_encode_valid()
218 static void its_encode_target(struct its_cmd_block *cmd, u64 target_addr)  in its_encode_target()  argument
220 	cmd->raw_cmd[2] &= ~(0xffffffffUL << 16);  in its_encode_target()
221 	cmd->raw_cmd[2] |= (target_addr & (0xffffffffUL << 16));  in its_encode_target()
224 static void its_encode_collection(struct its_cmd_block *cmd, u16 col)  in its_encode_collection()  argument
226 	cmd->raw_cmd[2] &= ~0xffffUL;  in its_encode_collection()
227 	cmd->raw_cmd[2] |= col;  in its_encode_collection()
230 static inline void its_fixup_cmd(struct its_cmd_block *cmd)  in its_fixup_cmd()  argument
233 	cmd->raw_cmd[0] = cpu_to_le64(cmd->raw_cmd[0]);  in its_fixup_cmd()
234 	cmd->raw_cmd[1] = cpu_to_le64(cmd->raw_cmd[1]);  in its_fixup_cmd()
235 	cmd->raw_cmd[2] = cpu_to_le64(cmd->raw_cmd[2]);  in its_fixup_cmd()
236 	cmd->raw_cmd[3] = cpu_to_le64(cmd->raw_cmd[3]);  in its_fixup_cmd()
239 static struct its_collection *its_build_mapd_cmd(struct its_cmd_block *cmd,  in its_build_mapd_cmd()  argument
248 	its_encode_cmd(cmd, GITS_CMD_MAPD);  in its_build_mapd_cmd()
249 	its_encode_devid(cmd, desc->its_mapd_cmd.dev->device_id);  in its_build_mapd_cmd()
250 	its_encode_size(cmd, size - 1);  in its_build_mapd_cmd()
251 	its_encode_itt(cmd, itt_addr);  in its_build_mapd_cmd()
252 	its_encode_valid(cmd, desc->its_mapd_cmd.valid);  in its_build_mapd_cmd()
254 	its_fixup_cmd(cmd);  in its_build_mapd_cmd()
259 static struct its_collection *its_build_mapc_cmd(struct its_cmd_block *cmd,  in its_build_mapc_cmd()  argument
262 	its_encode_cmd(cmd, GITS_CMD_MAPC);  in its_build_mapc_cmd()
263 	its_encode_collection(cmd, desc->its_mapc_cmd.col->col_id);  in its_build_mapc_cmd()
264 	its_encode_target(cmd, desc->its_mapc_cmd.col->target_address);  in its_build_mapc_cmd()
265 	its_encode_valid(cmd, desc->its_mapc_cmd.valid);  in its_build_mapc_cmd()
267 	its_fixup_cmd(cmd);  in its_build_mapc_cmd()
272 static struct its_collection *its_build_mapvi_cmd(struct its_cmd_block *cmd,  in its_build_mapvi_cmd()  argument
280 	its_encode_cmd(cmd, GITS_CMD_MAPVI);  in its_build_mapvi_cmd()
281 	its_encode_devid(cmd, desc->its_mapvi_cmd.dev->device_id);  in its_build_mapvi_cmd()
282 	its_encode_event_id(cmd, desc->its_mapvi_cmd.event_id);  in its_build_mapvi_cmd()
283 	its_encode_phys_id(cmd, desc->its_mapvi_cmd.phys_id);  in its_build_mapvi_cmd()
284 	its_encode_collection(cmd, col->col_id);  in its_build_mapvi_cmd()
286 	its_fixup_cmd(cmd);  in its_build_mapvi_cmd()
291 static struct its_collection *its_build_movi_cmd(struct its_cmd_block *cmd,  in its_build_movi_cmd()  argument
299 	its_encode_cmd(cmd, GITS_CMD_MOVI);  in its_build_movi_cmd()
300 	its_encode_devid(cmd, desc->its_movi_cmd.dev->device_id);  in its_build_movi_cmd()
301 	its_encode_event_id(cmd, desc->its_movi_cmd.event_id);  in its_build_movi_cmd()
302 	its_encode_collection(cmd, desc->its_movi_cmd.col->col_id);  in its_build_movi_cmd()
304 	its_fixup_cmd(cmd);  in its_build_movi_cmd()
309 static struct its_collection *its_build_discard_cmd(struct its_cmd_block *cmd,  in its_build_discard_cmd()  argument
317 	its_encode_cmd(cmd, GITS_CMD_DISCARD);  in its_build_discard_cmd()
318 	its_encode_devid(cmd, desc->its_discard_cmd.dev->device_id);  in its_build_discard_cmd()
319 	its_encode_event_id(cmd, desc->its_discard_cmd.event_id);  in its_build_discard_cmd()
321 	its_fixup_cmd(cmd);  in its_build_discard_cmd()
326 static struct its_collection *its_build_inv_cmd(struct its_cmd_block *cmd,  in its_build_inv_cmd()  argument
334 	its_encode_cmd(cmd, GITS_CMD_INV);  in its_build_inv_cmd()
335 	its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id);  in its_build_inv_cmd()
336 	its_encode_event_id(cmd, desc->its_inv_cmd.event_id);  in its_build_inv_cmd()
338 	its_fixup_cmd(cmd);  in its_build_inv_cmd()
343 static struct its_collection *its_build_invall_cmd(struct its_cmd_block *cmd,  in its_build_invall_cmd()  argument
346 	its_encode_cmd(cmd, GITS_CMD_INVALL);  in its_build_invall_cmd()
347 	its_encode_collection(cmd, desc->its_mapc_cmd.col->col_id);  in its_build_invall_cmd()
349 	its_fixup_cmd(cmd);  in its_build_invall_cmd()
377 	struct its_cmd_block *cmd;  in its_allocate_entry()  local
390 	cmd = its->cmd_write++;  in its_allocate_entry()
396 	return cmd;  in its_allocate_entry()
408 static void its_flush_cmd(struct its_node *its, struct its_cmd_block *cmd)  in its_flush_cmd()  argument
415 		__flush_dcache_area(cmd, sizeof(*cmd));  in its_flush_cmd()
449 	struct its_cmd_block *cmd, *sync_cmd, *next_cmd;  in its_send_single_command()  local
455 	cmd = its_allocate_entry(its);  in its_send_single_command()
456 	if (!cmd) {		/* We're soooooo screewed... */  in its_send_single_command()
461 	sync_col = builder(cmd, desc);  in its_send_single_command()
462 	its_flush_cmd(its, cmd);  in its_send_single_command()
480 	its_wait_for_range_completion(its, cmd, next_cmd);  in its_send_single_command()