45 static int iio_buffer_flush_hwfifo(struct iio_dev *indio_dev,  in iio_buffer_flush_hwfifo()  argument
48 	if (!indio_dev->info->hwfifo_flush_to_buffer)  in iio_buffer_flush_hwfifo()
51 	return indio_dev->info->hwfifo_flush_to_buffer(indio_dev, required);  in iio_buffer_flush_hwfifo()
54 static bool iio_buffer_ready(struct iio_dev *indio_dev, struct iio_buffer *buf,  in iio_buffer_ready()  argument
61 	if (!indio_dev->info)  in iio_buffer_ready()
75 			iio_buffer_flush_hwfifo(indio_dev, buf,  in iio_buffer_ready()
81 		flushed = iio_buffer_flush_hwfifo(indio_dev, buf,  in iio_buffer_ready()
108 	struct iio_dev *indio_dev = filp->private_data;  in iio_buffer_read_first_n_outer()  local
109 	struct iio_buffer *rb = indio_dev->buffer;  in iio_buffer_read_first_n_outer()
114 	if (!indio_dev->info)  in iio_buffer_read_first_n_outer()
136 		      iio_buffer_ready(indio_dev, rb, to_wait, n / datum_size));  in iio_buffer_read_first_n_outer()
140 		if (!indio_dev->info)  in iio_buffer_read_first_n_outer()
163 	struct iio_dev *indio_dev = filp->private_data;  in iio_buffer_poll()  local
164 	struct iio_buffer *rb = indio_dev->buffer;  in iio_buffer_poll()
166 	if (!indio_dev->info)  in iio_buffer_poll()
170 	if (iio_buffer_ready(indio_dev, rb, rb->watermark, 0))  in iio_buffer_poll()
182 void iio_buffer_wakeup_poll(struct iio_dev *indio_dev)  in iio_buffer_wakeup_poll()  argument
184 	if (!indio_dev->buffer)  in iio_buffer_wakeup_poll()
187 	wake_up(&indio_dev->buffer->pollq);  in iio_buffer_wakeup_poll()
243 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);  in iio_scan_el_show()  local
247 		       indio_dev->buffer->scan_mask);  in iio_scan_el_show()
273 static bool iio_validate_scan_mask(struct iio_dev *indio_dev,  in iio_validate_scan_mask()  argument
276 	if (!indio_dev->setup_ops->validate_scan_mask)  in iio_validate_scan_mask()
279 	return indio_dev->setup_ops->validate_scan_mask(indio_dev, mask);  in iio_validate_scan_mask()
292 static int iio_scan_mask_set(struct iio_dev *indio_dev,  in iio_scan_mask_set()  argument
299 			    BITS_TO_LONGS(indio_dev->masklength),  in iio_scan_mask_set()
304 	if (!indio_dev->masklength) {  in iio_scan_mask_set()
308 	bitmap_copy(trialmask, buffer->scan_mask, indio_dev->masklength);  in iio_scan_mask_set()
311 	if (!iio_validate_scan_mask(indio_dev, trialmask))  in iio_scan_mask_set()
314 	if (indio_dev->available_scan_masks) {  in iio_scan_mask_set()
315 		mask = iio_scan_mask_match(indio_dev->available_scan_masks,  in iio_scan_mask_set()
316 					   indio_dev->masklength,  in iio_scan_mask_set()
321 	bitmap_copy(buffer->scan_mask, trialmask, indio_dev->masklength);  in iio_scan_mask_set()
345 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);  in iio_scan_el_store()  local
346 	struct iio_buffer *buffer = indio_dev->buffer;  in iio_scan_el_store()
352 	mutex_lock(&indio_dev->mlock);  in iio_scan_el_store()
353 	if (iio_buffer_is_active(indio_dev->buffer)) {  in iio_scan_el_store()
357 	ret = iio_scan_mask_query(indio_dev, buffer, this_attr->address);  in iio_scan_el_store()
365 		ret = iio_scan_mask_set(indio_dev, buffer, this_attr->address);  in iio_scan_el_store()
371 	mutex_unlock(&indio_dev->mlock);  in iio_scan_el_store()
381 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);  in iio_scan_el_ts_show()  local
382 	return sprintf(buf, "%d\n", indio_dev->buffer->scan_timestamp);  in iio_scan_el_ts_show()
391 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);  in iio_scan_el_ts_store()  local
398 	mutex_lock(&indio_dev->mlock);  in iio_scan_el_ts_store()
399 	if (iio_buffer_is_active(indio_dev->buffer)) {  in iio_scan_el_ts_store()
403 	indio_dev->buffer->scan_timestamp = state;  in iio_scan_el_ts_store()
405 	mutex_unlock(&indio_dev->mlock);  in iio_scan_el_ts_store()
410 static int iio_buffer_add_channel_sysfs(struct iio_dev *indio_dev,  in iio_buffer_add_channel_sysfs()  argument
414 	struct iio_buffer *buffer = indio_dev->buffer;  in iio_buffer_add_channel_sysfs()
422 				     &indio_dev->dev,  in iio_buffer_add_channel_sysfs()
433 				     &indio_dev->dev,  in iio_buffer_add_channel_sysfs()
445 					     &indio_dev->dev,  in iio_buffer_add_channel_sysfs()
454 					     &indio_dev->dev,  in iio_buffer_add_channel_sysfs()
467 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);  in iio_buffer_read_length()  local
468 	struct iio_buffer *buffer = indio_dev->buffer;  in iio_buffer_read_length()
477 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);  in iio_buffer_write_length()  local
478 	struct iio_buffer *buffer = indio_dev->buffer;  in iio_buffer_write_length()
489 	mutex_lock(&indio_dev->mlock);  in iio_buffer_write_length()
490 	if (iio_buffer_is_active(indio_dev->buffer)) {  in iio_buffer_write_length()
501 	mutex_unlock(&indio_dev->mlock);  in iio_buffer_write_length()
510 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);  in iio_buffer_show_enable()  local
511 	return sprintf(buf, "%d\n", iio_buffer_is_active(indio_dev->buffer));  in iio_buffer_show_enable()
514 static int iio_compute_scan_bytes(struct iio_dev *indio_dev,  in iio_compute_scan_bytes()  argument
523 			 indio_dev->masklength) {  in iio_compute_scan_bytes()
524 		ch = iio_find_channel_from_si(indio_dev, i);  in iio_compute_scan_bytes()
534 		ch = iio_find_channel_from_si(indio_dev,  in iio_compute_scan_bytes()
535 					      indio_dev->scan_index_timestamp);  in iio_compute_scan_bytes()
547 static void iio_buffer_activate(struct iio_dev *indio_dev,  in iio_buffer_activate()  argument
551 	list_add(&buffer->buffer_list, &indio_dev->buffer_list);  in iio_buffer_activate()
561 static void iio_buffer_deactivate_all(struct iio_dev *indio_dev)  in iio_buffer_deactivate_all()  argument
566 			&indio_dev->buffer_list, buffer_list)  in iio_buffer_deactivate_all()
570 static void iio_buffer_update_bytes_per_datum(struct iio_dev *indio_dev,  in iio_buffer_update_bytes_per_datum()  argument
578 	bytes = iio_compute_scan_bytes(indio_dev, buffer->scan_mask,  in iio_buffer_update_bytes_per_datum()
584 static int iio_buffer_request_update(struct iio_dev *indio_dev,  in iio_buffer_request_update()  argument
589 	iio_buffer_update_bytes_per_datum(indio_dev, buffer);  in iio_buffer_request_update()
593 			dev_dbg(&indio_dev->dev,  in iio_buffer_request_update()
603 static void iio_free_scan_mask(struct iio_dev *indio_dev,  in iio_free_scan_mask()  argument
607 	if (!indio_dev->available_scan_masks)  in iio_free_scan_mask()
618 static int iio_verify_update(struct iio_dev *indio_dev,  in iio_verify_update()  argument
636 		list_is_singular(&indio_dev->buffer_list))  in iio_verify_update()
639 	modes = indio_dev->modes;  in iio_verify_update()
641 	list_for_each_entry(buffer, &indio_dev->buffer_list, buffer_list) {  in iio_verify_update()
651 	if ((modes & INDIO_BUFFER_TRIGGERED) && indio_dev->trig) {  in iio_verify_update()
658 		if (insert_buffer && !list_empty(&indio_dev->buffer_list))  in iio_verify_update()
666 		if (indio_dev->modes & INDIO_BUFFER_TRIGGERED)  in iio_verify_update()
667 			dev_dbg(&indio_dev->dev, "Buffer not started: no trigger\n");  in iio_verify_update()
672 	compound_mask = kcalloc(BITS_TO_LONGS(indio_dev->masklength),  in iio_verify_update()
679 	list_for_each_entry(buffer, &indio_dev->buffer_list, buffer_list) {  in iio_verify_update()
683 			  indio_dev->masklength);  in iio_verify_update()
689 			  insert_buffer->scan_mask, indio_dev->masklength);  in iio_verify_update()
693 	if (indio_dev->available_scan_masks) {  in iio_verify_update()
694 		scan_mask = iio_scan_mask_match(indio_dev->available_scan_masks,  in iio_verify_update()
695 				    indio_dev->masklength,  in iio_verify_update()
705 	config->scan_bytes = iio_compute_scan_bytes(indio_dev,  in iio_verify_update()
713 static int iio_enable_buffers(struct iio_dev *indio_dev,  in iio_enable_buffers()  argument
718 	indio_dev->active_scan_mask = config->scan_mask;  in iio_enable_buffers()
719 	indio_dev->scan_timestamp = config->scan_timestamp;  in iio_enable_buffers()
720 	indio_dev->scan_bytes = config->scan_bytes;  in iio_enable_buffers()
722 	iio_update_demux(indio_dev);  in iio_enable_buffers()
725 	if (indio_dev->setup_ops->preenable) {  in iio_enable_buffers()
726 		ret = indio_dev->setup_ops->preenable(indio_dev);  in iio_enable_buffers()
728 			dev_dbg(&indio_dev->dev,  in iio_enable_buffers()
734 	if (indio_dev->info->update_scan_mode) {  in iio_enable_buffers()
735 		ret = indio_dev->info  in iio_enable_buffers()
736 			->update_scan_mode(indio_dev,  in iio_enable_buffers()
737 					   indio_dev->active_scan_mask);  in iio_enable_buffers()
739 			dev_dbg(&indio_dev->dev,  in iio_enable_buffers()
746 	indio_dev->currentmode = config->mode;  in iio_enable_buffers()
748 	if (indio_dev->setup_ops->postenable) {  in iio_enable_buffers()
749 		ret = indio_dev->setup_ops->postenable(indio_dev);  in iio_enable_buffers()
751 			dev_dbg(&indio_dev->dev,  in iio_enable_buffers()
760 	indio_dev->currentmode = INDIO_DIRECT_MODE;  in iio_enable_buffers()
761 	if (indio_dev->setup_ops->postdisable)  in iio_enable_buffers()
762 		indio_dev->setup_ops->postdisable(indio_dev);  in iio_enable_buffers()
764 	indio_dev->active_scan_mask = NULL;  in iio_enable_buffers()
769 static int iio_disable_buffers(struct iio_dev *indio_dev)  in iio_disable_buffers()  argument
775 	if (list_empty(&indio_dev->buffer_list))  in iio_disable_buffers()
785 	if (indio_dev->setup_ops->predisable) {  in iio_disable_buffers()
786 		ret2 = indio_dev->setup_ops->predisable(indio_dev);  in iio_disable_buffers()
791 	indio_dev->currentmode = INDIO_DIRECT_MODE;  in iio_disable_buffers()
793 	if (indio_dev->setup_ops->postdisable) {  in iio_disable_buffers()
794 		ret2 = indio_dev->setup_ops->postdisable(indio_dev);  in iio_disable_buffers()
799 	iio_free_scan_mask(indio_dev, indio_dev->active_scan_mask);  in iio_disable_buffers()
800 	indio_dev->active_scan_mask = NULL;  in iio_disable_buffers()
805 static int __iio_update_buffers(struct iio_dev *indio_dev,  in __iio_update_buffers()  argument
812 	ret = iio_verify_update(indio_dev, insert_buffer, remove_buffer,  in __iio_update_buffers()
818 		ret = iio_buffer_request_update(indio_dev, insert_buffer);  in __iio_update_buffers()
823 	ret = iio_disable_buffers(indio_dev);  in __iio_update_buffers()
830 		iio_buffer_activate(indio_dev, insert_buffer);  in __iio_update_buffers()
833 	if (list_empty(&indio_dev->buffer_list))  in __iio_update_buffers()
836 	ret = iio_enable_buffers(indio_dev, &new_config);  in __iio_update_buffers()
851 	iio_buffer_deactivate_all(indio_dev);  in __iio_update_buffers()
854 	iio_free_scan_mask(indio_dev, new_config.scan_mask);  in __iio_update_buffers()
858 int iio_update_buffers(struct iio_dev *indio_dev,  in iio_update_buffers()  argument
867 	mutex_lock(&indio_dev->info_exist_lock);  in iio_update_buffers()
868 	mutex_lock(&indio_dev->mlock);  in iio_update_buffers()
881 	if (indio_dev->info == NULL) {  in iio_update_buffers()
886 	ret = __iio_update_buffers(indio_dev, insert_buffer, remove_buffer);  in iio_update_buffers()
889 	mutex_unlock(&indio_dev->mlock);  in iio_update_buffers()
890 	mutex_unlock(&indio_dev->info_exist_lock);  in iio_update_buffers()
896 void iio_disable_all_buffers(struct iio_dev *indio_dev)  in iio_disable_all_buffers()  argument
898 	iio_disable_buffers(indio_dev);  in iio_disable_all_buffers()
899 	iio_buffer_deactivate_all(indio_dev);  in iio_disable_all_buffers()
909 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);  in iio_buffer_store_enable()  local
916 	mutex_lock(&indio_dev->mlock);  in iio_buffer_store_enable()
919 	inlist = iio_buffer_is_active(indio_dev->buffer);  in iio_buffer_store_enable()
925 		ret = __iio_update_buffers(indio_dev,  in iio_buffer_store_enable()
926 					 indio_dev->buffer, NULL);  in iio_buffer_store_enable()
928 		ret = __iio_update_buffers(indio_dev,  in iio_buffer_store_enable()
929 					 NULL, indio_dev->buffer);  in iio_buffer_store_enable()
932 	mutex_unlock(&indio_dev->mlock);  in iio_buffer_store_enable()
942 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);  in iio_buffer_show_watermark()  local
943 	struct iio_buffer *buffer = indio_dev->buffer;  in iio_buffer_show_watermark()
953 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);  in iio_buffer_store_watermark()  local
954 	struct iio_buffer *buffer = indio_dev->buffer;  in iio_buffer_store_watermark()
964 	mutex_lock(&indio_dev->mlock);  in iio_buffer_store_watermark()
971 	if (iio_buffer_is_active(indio_dev->buffer)) {  in iio_buffer_store_watermark()
978 	if (indio_dev->info->hwfifo_set_watermark)  in iio_buffer_store_watermark()
979 		indio_dev->info->hwfifo_set_watermark(indio_dev, val);  in iio_buffer_store_watermark()
981 	mutex_unlock(&indio_dev->mlock);  in iio_buffer_store_watermark()
1001 int iio_buffer_alloc_sysfs_and_mask(struct iio_dev *indio_dev)  in iio_buffer_alloc_sysfs_and_mask()  argument
1005 	struct iio_buffer *buffer = indio_dev->buffer;  in iio_buffer_alloc_sysfs_and_mask()
1009 	channels = indio_dev->channels;  in iio_buffer_alloc_sysfs_and_mask()
1011 		int ml = indio_dev->masklength;  in iio_buffer_alloc_sysfs_and_mask()
1013 		for (i = 0; i < indio_dev->num_channels; i++)  in iio_buffer_alloc_sysfs_and_mask()
1015 		indio_dev->masklength = ml;  in iio_buffer_alloc_sysfs_and_mask()
1045 	indio_dev->groups[indio_dev->groupcounter++] = &buffer->buffer_group;  in iio_buffer_alloc_sysfs_and_mask()
1054 	channels = indio_dev->channels;  in iio_buffer_alloc_sysfs_and_mask()
1057 		for (i = 0; i < indio_dev->num_channels; i++) {  in iio_buffer_alloc_sysfs_and_mask()
1061 			ret = iio_buffer_add_channel_sysfs(indio_dev,  in iio_buffer_alloc_sysfs_and_mask()
1067 				indio_dev->scan_index_timestamp =  in iio_buffer_alloc_sysfs_and_mask()
1070 		if (indio_dev->masklength && buffer->scan_mask == NULL) {  in iio_buffer_alloc_sysfs_and_mask()
1071 			buffer->scan_mask = kcalloc(BITS_TO_LONGS(indio_dev->masklength),  in iio_buffer_alloc_sysfs_and_mask()
1097 	indio_dev->groups[indio_dev->groupcounter++] = &buffer->scan_el_group;  in iio_buffer_alloc_sysfs_and_mask()
1105 	kfree(indio_dev->buffer->buffer_group.attrs);  in iio_buffer_alloc_sysfs_and_mask()
1110 void iio_buffer_free_sysfs_and_mask(struct iio_dev *indio_dev)  in iio_buffer_free_sysfs_and_mask()  argument
1112 	if (!indio_dev->buffer)  in iio_buffer_free_sysfs_and_mask()
1115 	kfree(indio_dev->buffer->scan_mask);  in iio_buffer_free_sysfs_and_mask()
1116 	kfree(indio_dev->buffer->buffer_group.attrs);  in iio_buffer_free_sysfs_and_mask()
1117 	kfree(indio_dev->buffer->scan_el_group.attrs);  in iio_buffer_free_sysfs_and_mask()
1118 	iio_free_chan_devattr_list(&indio_dev->buffer->scan_el_dev_attr_list);  in iio_buffer_free_sysfs_and_mask()
1130 bool iio_validate_scan_mask_onehot(struct iio_dev *indio_dev,  in iio_validate_scan_mask_onehot()  argument
1133 	return bitmap_weight(mask, indio_dev->masklength) == 1;  in iio_validate_scan_mask_onehot()
1137 int iio_scan_mask_query(struct iio_dev *indio_dev,  in iio_scan_mask_query()  argument
1140 	if (bit > indio_dev->masklength)  in iio_scan_mask_query()
1206 int iio_push_to_buffers(struct iio_dev *indio_dev, const void *data)  in iio_push_to_buffers()  argument
1211 	list_for_each_entry(buf, &indio_dev->buffer_list, buffer_list) {  in iio_push_to_buffers()
1242 static int iio_buffer_update_demux(struct iio_dev *indio_dev,  in iio_buffer_update_demux()  argument
1256 	if (bitmap_equal(indio_dev->active_scan_mask,  in iio_buffer_update_demux()
1258 			 indio_dev->masklength))  in iio_buffer_update_demux()
1264 			 indio_dev->masklength) {  in iio_buffer_update_demux()
1265 		in_ind = find_next_bit(indio_dev->active_scan_mask,  in iio_buffer_update_demux()
1266 				       indio_dev->masklength,  in iio_buffer_update_demux()
1269 			in_ind = find_next_bit(indio_dev->active_scan_mask,  in iio_buffer_update_demux()
1270 					       indio_dev->masklength,  in iio_buffer_update_demux()
1272 			ch = iio_find_channel_from_si(indio_dev, in_ind);  in iio_buffer_update_demux()
1281 		ch = iio_find_channel_from_si(indio_dev, in_ind);  in iio_buffer_update_demux()
1297 		ch = iio_find_channel_from_si(indio_dev,  in iio_buffer_update_demux()
1298 			indio_dev->scan_index_timestamp);  in iio_buffer_update_demux()
1325 int iio_update_demux(struct iio_dev *indio_dev)  in iio_update_demux()  argument
1330 	list_for_each_entry(buffer, &indio_dev->buffer_list, buffer_list) {  in iio_update_demux()
1331 		ret = iio_buffer_update_demux(indio_dev, buffer);  in iio_update_demux()
1338 	list_for_each_entry(buffer, &indio_dev->buffer_list, buffer_list)  in iio_update_demux()