40 static inline size_t buffer_size(struct persistent_ram_zone *prz)  in buffer_size()  argument
42 	return atomic_read(&prz->buffer->size);  in buffer_size()
45 static inline size_t buffer_start(struct persistent_ram_zone *prz)  in buffer_start()  argument
47 	return atomic_read(&prz->buffer->start);  in buffer_start()
51 static size_t buffer_start_add_atomic(struct persistent_ram_zone *prz, size_t a)  in buffer_start_add_atomic()  argument
57 		old = atomic_read(&prz->buffer->start);  in buffer_start_add_atomic()
59 		while (unlikely(new >= prz->buffer_size))  in buffer_start_add_atomic()
60 			new -= prz->buffer_size;  in buffer_start_add_atomic()
61 	} while (atomic_cmpxchg(&prz->buffer->start, old, new) != old);  in buffer_start_add_atomic()
67 static void buffer_size_add_atomic(struct persistent_ram_zone *prz, size_t a)  in buffer_size_add_atomic()  argument
72 	if (atomic_read(&prz->buffer->size) == prz->buffer_size)  in buffer_size_add_atomic()
76 		old = atomic_read(&prz->buffer->size);  in buffer_size_add_atomic()
78 		if (new > prz->buffer_size)  in buffer_size_add_atomic()
79 			new = prz->buffer_size;  in buffer_size_add_atomic()
80 	} while (atomic_cmpxchg(&prz->buffer->size, old, new) != old);  in buffer_size_add_atomic()
86 static size_t buffer_start_add_locked(struct persistent_ram_zone *prz, size_t a)  in buffer_start_add_locked()  argument
94 	old = atomic_read(&prz->buffer->start);  in buffer_start_add_locked()
96 	while (unlikely(new >= prz->buffer_size))  in buffer_start_add_locked()
97 		new -= prz->buffer_size;  in buffer_start_add_locked()
98 	atomic_set(&prz->buffer->start, new);  in buffer_start_add_locked()
106 static void buffer_size_add_locked(struct persistent_ram_zone *prz, size_t a)  in buffer_size_add_locked()  argument
114 	old = atomic_read(&prz->buffer->size);  in buffer_size_add_locked()
115 	if (old == prz->buffer_size)  in buffer_size_add_locked()
119 	if (new > prz->buffer_size)  in buffer_size_add_locked()
120 		new = prz->buffer_size;  in buffer_size_add_locked()
121 	atomic_set(&prz->buffer->size, new);  in buffer_size_add_locked()
130 static void notrace persistent_ram_encode_rs8(struct persistent_ram_zone *prz,  in persistent_ram_encode_rs8()  argument
134 	uint16_t par[prz->ecc_info.ecc_size];  in persistent_ram_encode_rs8()
138 	encode_rs8(prz->rs_decoder, data, len, par, 0);  in persistent_ram_encode_rs8()
139 	for (i = 0; i < prz->ecc_info.ecc_size; i++)  in persistent_ram_encode_rs8()
143 static int persistent_ram_decode_rs8(struct persistent_ram_zone *prz,  in persistent_ram_decode_rs8()  argument
147 	uint16_t par[prz->ecc_info.ecc_size];  in persistent_ram_decode_rs8()
149 	for (i = 0; i < prz->ecc_info.ecc_size; i++)  in persistent_ram_decode_rs8()
151 	return decode_rs8(prz->rs_decoder, data, par, len,  in persistent_ram_decode_rs8()
155 static void notrace persistent_ram_update_ecc(struct persistent_ram_zone *prz,  in persistent_ram_update_ecc()  argument
158 	struct persistent_ram_buffer *buffer = prz->buffer;  in persistent_ram_update_ecc()
159 	uint8_t *buffer_end = buffer->data + prz->buffer_size;  in persistent_ram_update_ecc()
162 	int ecc_block_size = prz->ecc_info.block_size;  in persistent_ram_update_ecc()
163 	int ecc_size = prz->ecc_info.ecc_size;  in persistent_ram_update_ecc()
170 	par = prz->par_buffer + (start / ecc_block_size) * ecc_size;  in persistent_ram_update_ecc()
175 		persistent_ram_encode_rs8(prz, block, size, par);  in persistent_ram_update_ecc()
181 static void persistent_ram_update_header_ecc(struct persistent_ram_zone *prz)  in persistent_ram_update_header_ecc()  argument
183 	struct persistent_ram_buffer *buffer = prz->buffer;  in persistent_ram_update_header_ecc()
185 	if (!prz->ecc_info.ecc_size)  in persistent_ram_update_header_ecc()
188 	persistent_ram_encode_rs8(prz, (uint8_t *)buffer, sizeof(*buffer),  in persistent_ram_update_header_ecc()
189 				  prz->par_header);  in persistent_ram_update_header_ecc()
192 static void persistent_ram_ecc_old(struct persistent_ram_zone *prz)  in persistent_ram_ecc_old()  argument
194 	struct persistent_ram_buffer *buffer = prz->buffer;  in persistent_ram_ecc_old()
198 	if (!prz->ecc_info.ecc_size)  in persistent_ram_ecc_old()
202 	par = prz->par_buffer;  in persistent_ram_ecc_old()
203 	while (block < buffer->data + buffer_size(prz)) {  in persistent_ram_ecc_old()
205 		int size = prz->ecc_info.block_size;  in persistent_ram_ecc_old()
206 		if (block + size > buffer->data + prz->buffer_size)  in persistent_ram_ecc_old()
207 			size = buffer->data + prz->buffer_size - block;  in persistent_ram_ecc_old()
208 		numerr = persistent_ram_decode_rs8(prz, block, size, par);  in persistent_ram_ecc_old()
211 			prz->corrected_bytes += numerr;  in persistent_ram_ecc_old()
214 			prz->bad_blocks++;  in persistent_ram_ecc_old()
216 		block += prz->ecc_info.block_size;  in persistent_ram_ecc_old()
217 		par += prz->ecc_info.ecc_size;  in persistent_ram_ecc_old()
221 static int persistent_ram_init_ecc(struct persistent_ram_zone *prz,  in persistent_ram_init_ecc()  argument
225 	struct persistent_ram_buffer *buffer = prz->buffer;  in persistent_ram_init_ecc()
232 	prz->ecc_info.block_size = ecc_info->block_size ?: 128;  in persistent_ram_init_ecc()
233 	prz->ecc_info.ecc_size = ecc_info->ecc_size ?: 16;  in persistent_ram_init_ecc()
234 	prz->ecc_info.symsize = ecc_info->symsize ?: 8;  in persistent_ram_init_ecc()
235 	prz->ecc_info.poly = ecc_info->poly ?: 0x11d;  in persistent_ram_init_ecc()
237 	ecc_blocks = DIV_ROUND_UP(prz->buffer_size - prz->ecc_info.ecc_size,  in persistent_ram_init_ecc()
238 				  prz->ecc_info.block_size +  in persistent_ram_init_ecc()
239 				  prz->ecc_info.ecc_size);  in persistent_ram_init_ecc()
240 	ecc_total = (ecc_blocks + 1) * prz->ecc_info.ecc_size;  in persistent_ram_init_ecc()
241 	if (ecc_total >= prz->buffer_size) {  in persistent_ram_init_ecc()
243 		       __func__, prz->ecc_info.ecc_size,  in persistent_ram_init_ecc()
244 		       ecc_total, prz->buffer_size);  in persistent_ram_init_ecc()
248 	prz->buffer_size -= ecc_total;  in persistent_ram_init_ecc()
249 	prz->par_buffer = buffer->data + prz->buffer_size;  in persistent_ram_init_ecc()
250 	prz->par_header = prz->par_buffer +  in persistent_ram_init_ecc()
251 			  ecc_blocks * prz->ecc_info.ecc_size;  in persistent_ram_init_ecc()
257 	prz->rs_decoder = init_rs(prz->ecc_info.symsize, prz->ecc_info.poly,  in persistent_ram_init_ecc()
258 				  0, 1, prz->ecc_info.ecc_size);  in persistent_ram_init_ecc()
259 	if (prz->rs_decoder == NULL) {  in persistent_ram_init_ecc()
264 	prz->corrected_bytes = 0;  in persistent_ram_init_ecc()
265 	prz->bad_blocks = 0;  in persistent_ram_init_ecc()
267 	numerr = persistent_ram_decode_rs8(prz, buffer, sizeof(*buffer),  in persistent_ram_init_ecc()
268 					   prz->par_header);  in persistent_ram_init_ecc()
271 		prz->corrected_bytes += numerr;  in persistent_ram_init_ecc()
274 		prz->bad_blocks++;  in persistent_ram_init_ecc()
280 ssize_t persistent_ram_ecc_string(struct persistent_ram_zone *prz,  in persistent_ram_ecc_string()  argument
285 	if (!prz->ecc_info.ecc_size)  in persistent_ram_ecc_string()
288 	if (prz->corrected_bytes || prz->bad_blocks)  in persistent_ram_ecc_string()
291 			prz->corrected_bytes, prz->bad_blocks);  in persistent_ram_ecc_string()
298 static void notrace persistent_ram_update(struct persistent_ram_zone *prz,  in persistent_ram_update()  argument
301 	struct persistent_ram_buffer *buffer = prz->buffer;  in persistent_ram_update()
303 	persistent_ram_update_ecc(prz, start, count);  in persistent_ram_update()
306 void persistent_ram_save_old(struct persistent_ram_zone *prz)  in persistent_ram_save_old()  argument
308 	struct persistent_ram_buffer *buffer = prz->buffer;  in persistent_ram_save_old()
309 	size_t size = buffer_size(prz);  in persistent_ram_save_old()
310 	size_t start = buffer_start(prz);  in persistent_ram_save_old()
315 	if (!prz->old_log) {  in persistent_ram_save_old()
316 		persistent_ram_ecc_old(prz);  in persistent_ram_save_old()
317 		prz->old_log = kmalloc(size, GFP_KERNEL);  in persistent_ram_save_old()
319 	if (!prz->old_log) {  in persistent_ram_save_old()
324 	prz->old_log_size = size;  in persistent_ram_save_old()
325 	memcpy(prz->old_log, &buffer->data[start], size - start);  in persistent_ram_save_old()
326 	memcpy(prz->old_log + size - start, &buffer->data[0], start);  in persistent_ram_save_old()
329 int notrace persistent_ram_write(struct persistent_ram_zone *prz,  in persistent_ram_write()  argument
336 	if (unlikely(c > prz->buffer_size)) {  in persistent_ram_write()
337 		s += c - prz->buffer_size;  in persistent_ram_write()
338 		c = prz->buffer_size;  in persistent_ram_write()
341 	buffer_size_add(prz, c);  in persistent_ram_write()
343 	start = buffer_start_add(prz, c);  in persistent_ram_write()
345 	rem = prz->buffer_size - start;  in persistent_ram_write()
347 		persistent_ram_update(prz, s, start, rem);  in persistent_ram_write()
352 	persistent_ram_update(prz, s, start, c);  in persistent_ram_write()
354 	persistent_ram_update_header_ecc(prz);  in persistent_ram_write()
359 size_t persistent_ram_old_size(struct persistent_ram_zone *prz)  in persistent_ram_old_size()  argument
361 	return prz->old_log_size;  in persistent_ram_old_size()
364 void *persistent_ram_old(struct persistent_ram_zone *prz)  in persistent_ram_old()  argument
366 	return prz->old_log;  in persistent_ram_old()
369 void persistent_ram_free_old(struct persistent_ram_zone *prz)  in persistent_ram_free_old()  argument
371 	kfree(prz->old_log);  in persistent_ram_free_old()
372 	prz->old_log = NULL;  in persistent_ram_free_old()
373 	prz->old_log_size = 0;  in persistent_ram_free_old()
376 void persistent_ram_zap(struct persistent_ram_zone *prz)  in persistent_ram_zap()  argument
378 	atomic_set(&prz->buffer->start, 0);  in persistent_ram_zap()
379 	atomic_set(&prz->buffer->size, 0);  in persistent_ram_zap()
380 	persistent_ram_update_header_ecc(prz);  in persistent_ram_zap()
441 		struct persistent_ram_zone *prz, int memtype)  in persistent_ram_buffer_map()  argument
443 	prz->paddr = start;  in persistent_ram_buffer_map()
444 	prz->size = size;  in persistent_ram_buffer_map()
447 		prz->vaddr = persistent_ram_vmap(start, size, memtype);  in persistent_ram_buffer_map()
449 		prz->vaddr = persistent_ram_iomap(start, size, memtype);  in persistent_ram_buffer_map()
451 	if (!prz->vaddr) {  in persistent_ram_buffer_map()
457 	prz->buffer = prz->vaddr + offset_in_page(start);  in persistent_ram_buffer_map()
458 	prz->buffer_size = size - sizeof(struct persistent_ram_buffer);  in persistent_ram_buffer_map()
463 static int persistent_ram_post_init(struct persistent_ram_zone *prz, u32 sig,  in persistent_ram_post_init()  argument
468 	ret = persistent_ram_init_ecc(prz, ecc_info);  in persistent_ram_post_init()
474 	if (prz->buffer->sig == sig) {  in persistent_ram_post_init()
475 		if (buffer_size(prz) > prz->buffer_size ||  in persistent_ram_post_init()
476 		    buffer_start(prz) > buffer_size(prz))  in persistent_ram_post_init()
478 				buffer_size(prz), buffer_start(prz));  in persistent_ram_post_init()
481 				 buffer_size(prz), buffer_start(prz));  in persistent_ram_post_init()
482 			persistent_ram_save_old(prz);  in persistent_ram_post_init()
487 			 prz->buffer->sig);  in persistent_ram_post_init()
490 	prz->buffer->sig = sig;  in persistent_ram_post_init()
491 	persistent_ram_zap(prz);  in persistent_ram_post_init()
496 void persistent_ram_free(struct persistent_ram_zone *prz)  in persistent_ram_free()  argument
498 	if (!prz)  in persistent_ram_free()
501 	if (prz->vaddr) {  in persistent_ram_free()
502 		if (pfn_valid(prz->paddr >> PAGE_SHIFT)) {  in persistent_ram_free()
503 			vunmap(prz->vaddr);  in persistent_ram_free()
505 			iounmap(prz->vaddr);  in persistent_ram_free()
506 			release_mem_region(prz->paddr, prz->size);  in persistent_ram_free()
508 		prz->vaddr = NULL;  in persistent_ram_free()
510 	persistent_ram_free_old(prz);  in persistent_ram_free()
511 	kfree(prz);  in persistent_ram_free()
518 	struct persistent_ram_zone *prz;  in persistent_ram_new()  local
521 	prz = kzalloc(sizeof(struct persistent_ram_zone), GFP_KERNEL);  in persistent_ram_new()
522 	if (!prz) {  in persistent_ram_new()
527 	ret = persistent_ram_buffer_map(start, size, prz, memtype);  in persistent_ram_new()
531 	ret = persistent_ram_post_init(prz, sig, ecc_info);  in persistent_ram_new()
535 	return prz;  in persistent_ram_new()
537 	persistent_ram_free(prz);  in persistent_ram_new()