744 static int queue_insert_raw(struct arm_smmu_queue *q, u64 *ent)  in queue_insert_raw()  argument
749 	queue_write(Q_ENT(q, q->prod), ent, q->ent_dwords);  in queue_insert_raw()
762 static int queue_remove_raw(struct arm_smmu_queue *q, u64 *ent)  in queue_remove_raw()  argument
767 	queue_read(ent, Q_ENT(q, q->cons), q->ent_dwords);  in queue_remove_raw()
773 static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct arm_smmu_cmdq_ent *ent)  in arm_smmu_cmdq_build_cmd()  argument
776 	cmd[0] |= (ent->opcode & CMDQ_0_OP_MASK) << CMDQ_0_OP_SHIFT;  in arm_smmu_cmdq_build_cmd()
778 	switch (ent->opcode) {  in arm_smmu_cmdq_build_cmd()
783 		cmd[0] |= (u64)ent->prefetch.sid << CMDQ_PREFETCH_0_SID_SHIFT;  in arm_smmu_cmdq_build_cmd()
784 		cmd[1] |= ent->prefetch.size << CMDQ_PREFETCH_1_SIZE_SHIFT;  in arm_smmu_cmdq_build_cmd()
785 		cmd[1] |= ent->prefetch.addr & CMDQ_PREFETCH_1_ADDR_MASK;  in arm_smmu_cmdq_build_cmd()
788 		cmd[0] |= (u64)ent->cfgi.sid << CMDQ_CFGI_0_SID_SHIFT;  in arm_smmu_cmdq_build_cmd()
789 		cmd[1] |= ent->cfgi.leaf ? CMDQ_CFGI_1_LEAF : 0;  in arm_smmu_cmdq_build_cmd()
796 		cmd[0] |= (u64)ent->tlbi.asid << CMDQ_TLBI_0_ASID_SHIFT;  in arm_smmu_cmdq_build_cmd()
797 		cmd[1] |= ent->tlbi.leaf ? CMDQ_TLBI_1_LEAF : 0;  in arm_smmu_cmdq_build_cmd()
798 		cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_VA_MASK;  in arm_smmu_cmdq_build_cmd()
801 		cmd[0] |= (u64)ent->tlbi.vmid << CMDQ_TLBI_0_VMID_SHIFT;  in arm_smmu_cmdq_build_cmd()
802 		cmd[1] |= ent->tlbi.leaf ? CMDQ_TLBI_1_LEAF : 0;  in arm_smmu_cmdq_build_cmd()
803 		cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_IPA_MASK;  in arm_smmu_cmdq_build_cmd()
806 		cmd[0] |= (u64)ent->tlbi.asid << CMDQ_TLBI_0_ASID_SHIFT;  in arm_smmu_cmdq_build_cmd()
809 		cmd[0] |= (u64)ent->tlbi.vmid << CMDQ_TLBI_0_VMID_SHIFT;  in arm_smmu_cmdq_build_cmd()
812 		cmd[0] |= ent->substream_valid ? CMDQ_0_SSV : 0;  in arm_smmu_cmdq_build_cmd()
813 		cmd[0] |= ent->pri.ssid << CMDQ_PRI_0_SSID_SHIFT;  in arm_smmu_cmdq_build_cmd()
814 		cmd[0] |= (u64)ent->pri.sid << CMDQ_PRI_0_SID_SHIFT;  in arm_smmu_cmdq_build_cmd()
815 		cmd[1] |= ent->pri.grpid << CMDQ_PRI_1_GRPID_SHIFT;  in arm_smmu_cmdq_build_cmd()
816 		switch (ent->pri.resp) {  in arm_smmu_cmdq_build_cmd()
888 				    struct arm_smmu_cmdq_ent *ent)  in arm_smmu_cmdq_issue_cmd()  argument
895 	if (arm_smmu_cmdq_build_cmd(cmd, ent)) {  in arm_smmu_cmdq_issue_cmd()
897 			 ent->opcode);  in arm_smmu_cmdq_issue_cmd()
912 	if (ent->opcode == CMDQ_OP_CMD_SYNC && queue_poll_cons(q, until, wfe))  in arm_smmu_cmdq_issue_cmd()