This source file includes following definitions.
- z_erofs_fill_inode
- fill_inode_lazy
- z_erofs_reload_indexes
- vle_legacy_load_cluster_from_disk
- decode_compactedbits
- unpack_compacted_index
- compacted_load_cluster_from_disk
- vle_load_cluster_from_disk
- vle_extent_lookback
- z_erofs_map_blocks_iter
1
2
3
4
5
6
7 #include "internal.h"
8 #include <asm/unaligned.h>
9 #include <trace/events/erofs.h>
10
11 int z_erofs_fill_inode(struct inode *inode)
12 {
13 struct erofs_inode *const vi = EROFS_I(inode);
14
15 if (vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY) {
16 vi->z_advise = 0;
17 vi->z_algorithmtype[0] = 0;
18 vi->z_algorithmtype[1] = 0;
19 vi->z_logical_clusterbits = LOG_BLOCK_SIZE;
20 vi->z_physical_clusterbits[0] = vi->z_logical_clusterbits;
21 vi->z_physical_clusterbits[1] = vi->z_logical_clusterbits;
22 set_bit(EROFS_I_Z_INITED_BIT, &vi->flags);
23 }
24
25 inode->i_mapping->a_ops = &z_erofs_vle_normalaccess_aops;
26 return 0;
27 }
28
29 static int fill_inode_lazy(struct inode *inode)
30 {
31 struct erofs_inode *const vi = EROFS_I(inode);
32 struct super_block *const sb = inode->i_sb;
33 int err;
34 erofs_off_t pos;
35 struct page *page;
36 void *kaddr;
37 struct z_erofs_map_header *h;
38
39 if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags))
40 return 0;
41
42 if (wait_on_bit_lock(&vi->flags, EROFS_I_BL_Z_BIT, TASK_KILLABLE))
43 return -ERESTARTSYS;
44
45 err = 0;
46 if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags))
47 goto out_unlock;
48
49 DBG_BUGON(vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY);
50
51 pos = ALIGN(iloc(EROFS_SB(sb), vi->nid) + vi->inode_isize +
52 vi->xattr_isize, 8);
53 page = erofs_get_meta_page(sb, erofs_blknr(pos));
54 if (IS_ERR(page)) {
55 err = PTR_ERR(page);
56 goto out_unlock;
57 }
58
59 kaddr = kmap_atomic(page);
60
61 h = kaddr + erofs_blkoff(pos);
62 vi->z_advise = le16_to_cpu(h->h_advise);
63 vi->z_algorithmtype[0] = h->h_algorithmtype & 15;
64 vi->z_algorithmtype[1] = h->h_algorithmtype >> 4;
65
66 if (vi->z_algorithmtype[0] >= Z_EROFS_COMPRESSION_MAX) {
67 erofs_err(sb, "unknown compression format %u for nid %llu, please upgrade kernel",
68 vi->z_algorithmtype[0], vi->nid);
69 err = -EOPNOTSUPP;
70 goto unmap_done;
71 }
72
73 vi->z_logical_clusterbits = LOG_BLOCK_SIZE + (h->h_clusterbits & 7);
74 vi->z_physical_clusterbits[0] = vi->z_logical_clusterbits +
75 ((h->h_clusterbits >> 3) & 3);
76
77 if (vi->z_physical_clusterbits[0] != LOG_BLOCK_SIZE) {
78 erofs_err(sb, "unsupported physical clusterbits %u for nid %llu, please upgrade kernel",
79 vi->z_physical_clusterbits[0], vi->nid);
80 err = -EOPNOTSUPP;
81 goto unmap_done;
82 }
83
84 vi->z_physical_clusterbits[1] = vi->z_logical_clusterbits +
85 ((h->h_clusterbits >> 5) & 7);
86 set_bit(EROFS_I_Z_INITED_BIT, &vi->flags);
87 unmap_done:
88 kunmap_atomic(kaddr);
89 unlock_page(page);
90 put_page(page);
91 out_unlock:
92 clear_and_wake_up_bit(EROFS_I_BL_Z_BIT, &vi->flags);
93 return err;
94 }
95
96 struct z_erofs_maprecorder {
97 struct inode *inode;
98 struct erofs_map_blocks *map;
99 void *kaddr;
100
101 unsigned long lcn;
102
103 u8 type;
104 u16 clusterofs;
105 u16 delta[2];
106 erofs_blk_t pblk;
107 };
108
109 static int z_erofs_reload_indexes(struct z_erofs_maprecorder *m,
110 erofs_blk_t eblk)
111 {
112 struct super_block *const sb = m->inode->i_sb;
113 struct erofs_map_blocks *const map = m->map;
114 struct page *mpage = map->mpage;
115
116 if (mpage) {
117 if (mpage->index == eblk) {
118 if (!m->kaddr)
119 m->kaddr = kmap_atomic(mpage);
120 return 0;
121 }
122
123 if (m->kaddr) {
124 kunmap_atomic(m->kaddr);
125 m->kaddr = NULL;
126 }
127 put_page(mpage);
128 }
129
130 mpage = erofs_get_meta_page(sb, eblk);
131 if (IS_ERR(mpage)) {
132 map->mpage = NULL;
133 return PTR_ERR(mpage);
134 }
135 m->kaddr = kmap_atomic(mpage);
136 unlock_page(mpage);
137 map->mpage = mpage;
138 return 0;
139 }
140
141 static int vle_legacy_load_cluster_from_disk(struct z_erofs_maprecorder *m,
142 unsigned long lcn)
143 {
144 struct inode *const inode = m->inode;
145 struct erofs_inode *const vi = EROFS_I(inode);
146 const erofs_off_t ibase = iloc(EROFS_I_SB(inode), vi->nid);
147 const erofs_off_t pos =
148 Z_EROFS_VLE_LEGACY_INDEX_ALIGN(ibase + vi->inode_isize +
149 vi->xattr_isize) +
150 lcn * sizeof(struct z_erofs_vle_decompressed_index);
151 struct z_erofs_vle_decompressed_index *di;
152 unsigned int advise, type;
153 int err;
154
155 err = z_erofs_reload_indexes(m, erofs_blknr(pos));
156 if (err)
157 return err;
158
159 m->lcn = lcn;
160 di = m->kaddr + erofs_blkoff(pos);
161
162 advise = le16_to_cpu(di->di_advise);
163 type = (advise >> Z_EROFS_VLE_DI_CLUSTER_TYPE_BIT) &
164 ((1 << Z_EROFS_VLE_DI_CLUSTER_TYPE_BITS) - 1);
165 switch (type) {
166 case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD:
167 m->clusterofs = 1 << vi->z_logical_clusterbits;
168 m->delta[0] = le16_to_cpu(di->di_u.delta[0]);
169 m->delta[1] = le16_to_cpu(di->di_u.delta[1]);
170 break;
171 case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN:
172 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD:
173 m->clusterofs = le16_to_cpu(di->di_clusterofs);
174 m->pblk = le32_to_cpu(di->di_u.blkaddr);
175 break;
176 default:
177 DBG_BUGON(1);
178 return -EOPNOTSUPP;
179 }
180 m->type = type;
181 return 0;
182 }
183
184 static unsigned int decode_compactedbits(unsigned int lobits,
185 unsigned int lomask,
186 u8 *in, unsigned int pos, u8 *type)
187 {
188 const unsigned int v = get_unaligned_le32(in + pos / 8) >> (pos & 7);
189 const unsigned int lo = v & lomask;
190
191 *type = (v >> lobits) & 3;
192 return lo;
193 }
194
195 static int unpack_compacted_index(struct z_erofs_maprecorder *m,
196 unsigned int amortizedshift,
197 unsigned int eofs)
198 {
199 struct erofs_inode *const vi = EROFS_I(m->inode);
200 const unsigned int lclusterbits = vi->z_logical_clusterbits;
201 const unsigned int lomask = (1 << lclusterbits) - 1;
202 unsigned int vcnt, base, lo, encodebits, nblk;
203 int i;
204 u8 *in, type;
205
206 if (1 << amortizedshift == 4)
207 vcnt = 2;
208 else if (1 << amortizedshift == 2 && lclusterbits == 12)
209 vcnt = 16;
210 else
211 return -EOPNOTSUPP;
212
213 encodebits = ((vcnt << amortizedshift) - sizeof(__le32)) * 8 / vcnt;
214 base = round_down(eofs, vcnt << amortizedshift);
215 in = m->kaddr + base;
216
217 i = (eofs - base) >> amortizedshift;
218
219 lo = decode_compactedbits(lclusterbits, lomask,
220 in, encodebits * i, &type);
221 m->type = type;
222 if (type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) {
223 m->clusterofs = 1 << lclusterbits;
224 if (i + 1 != vcnt) {
225 m->delta[0] = lo;
226 return 0;
227 }
228
229
230
231
232
233 lo = decode_compactedbits(lclusterbits, lomask,
234 in, encodebits * (i - 1), &type);
235 if (type != Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD)
236 lo = 0;
237 m->delta[0] = lo + 1;
238 return 0;
239 }
240 m->clusterofs = lo;
241 m->delta[0] = 0;
242
243 nblk = 1;
244 while (i > 0) {
245 --i;
246 lo = decode_compactedbits(lclusterbits, lomask,
247 in, encodebits * i, &type);
248 if (type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD)
249 i -= lo;
250
251 if (i >= 0)
252 ++nblk;
253 }
254 in += (vcnt << amortizedshift) - sizeof(__le32);
255 m->pblk = le32_to_cpu(*(__le32 *)in) + nblk;
256 return 0;
257 }
258
259 static int compacted_load_cluster_from_disk(struct z_erofs_maprecorder *m,
260 unsigned long lcn)
261 {
262 struct inode *const inode = m->inode;
263 struct erofs_inode *const vi = EROFS_I(inode);
264 const unsigned int lclusterbits = vi->z_logical_clusterbits;
265 const erofs_off_t ebase = ALIGN(iloc(EROFS_I_SB(inode), vi->nid) +
266 vi->inode_isize + vi->xattr_isize, 8) +
267 sizeof(struct z_erofs_map_header);
268 const unsigned int totalidx = DIV_ROUND_UP(inode->i_size, EROFS_BLKSIZ);
269 unsigned int compacted_4b_initial, compacted_2b;
270 unsigned int amortizedshift;
271 erofs_off_t pos;
272 int err;
273
274 if (lclusterbits != 12)
275 return -EOPNOTSUPP;
276
277 if (lcn >= totalidx)
278 return -EINVAL;
279
280 m->lcn = lcn;
281
282 compacted_4b_initial = (32 - ebase % 32) / 4;
283 if (compacted_4b_initial == 32 / 4)
284 compacted_4b_initial = 0;
285
286 if (vi->z_advise & Z_EROFS_ADVISE_COMPACTED_2B)
287 compacted_2b = rounddown(totalidx - compacted_4b_initial, 16);
288 else
289 compacted_2b = 0;
290
291 pos = ebase;
292 if (lcn < compacted_4b_initial) {
293 amortizedshift = 2;
294 goto out;
295 }
296 pos += compacted_4b_initial * 4;
297 lcn -= compacted_4b_initial;
298
299 if (lcn < compacted_2b) {
300 amortizedshift = 1;
301 goto out;
302 }
303 pos += compacted_2b * 2;
304 lcn -= compacted_2b;
305 amortizedshift = 2;
306 out:
307 pos += lcn * (1 << amortizedshift);
308 err = z_erofs_reload_indexes(m, erofs_blknr(pos));
309 if (err)
310 return err;
311 return unpack_compacted_index(m, amortizedshift, erofs_blkoff(pos));
312 }
313
314 static int vle_load_cluster_from_disk(struct z_erofs_maprecorder *m,
315 unsigned int lcn)
316 {
317 const unsigned int datamode = EROFS_I(m->inode)->datalayout;
318
319 if (datamode == EROFS_INODE_FLAT_COMPRESSION_LEGACY)
320 return vle_legacy_load_cluster_from_disk(m, lcn);
321
322 if (datamode == EROFS_INODE_FLAT_COMPRESSION)
323 return compacted_load_cluster_from_disk(m, lcn);
324
325 return -EINVAL;
326 }
327
328 static int vle_extent_lookback(struct z_erofs_maprecorder *m,
329 unsigned int lookback_distance)
330 {
331 struct erofs_inode *const vi = EROFS_I(m->inode);
332 struct erofs_map_blocks *const map = m->map;
333 const unsigned int lclusterbits = vi->z_logical_clusterbits;
334 unsigned long lcn = m->lcn;
335 int err;
336
337 if (lcn < lookback_distance) {
338 erofs_err(m->inode->i_sb,
339 "bogus lookback distance @ nid %llu", vi->nid);
340 DBG_BUGON(1);
341 return -EFSCORRUPTED;
342 }
343
344
345 lcn -= lookback_distance;
346 err = vle_load_cluster_from_disk(m, lcn);
347 if (err)
348 return err;
349
350 switch (m->type) {
351 case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD:
352 if (!m->delta[0]) {
353 erofs_err(m->inode->i_sb,
354 "invalid lookback distance 0 @ nid %llu",
355 vi->nid);
356 DBG_BUGON(1);
357 return -EFSCORRUPTED;
358 }
359 return vle_extent_lookback(m, m->delta[0]);
360 case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN:
361 map->m_flags &= ~EROFS_MAP_ZIPPED;
362
363 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD:
364 map->m_la = (lcn << lclusterbits) | m->clusterofs;
365 break;
366 default:
367 erofs_err(m->inode->i_sb,
368 "unknown type %u @ lcn %lu of nid %llu",
369 m->type, lcn, vi->nid);
370 DBG_BUGON(1);
371 return -EOPNOTSUPP;
372 }
373 return 0;
374 }
375
376 int z_erofs_map_blocks_iter(struct inode *inode,
377 struct erofs_map_blocks *map,
378 int flags)
379 {
380 struct erofs_inode *const vi = EROFS_I(inode);
381 struct z_erofs_maprecorder m = {
382 .inode = inode,
383 .map = map,
384 };
385 int err = 0;
386 unsigned int lclusterbits, endoff;
387 unsigned long long ofs, end;
388
389 trace_z_erofs_map_blocks_iter_enter(inode, map, flags);
390
391
392 if (map->m_la >= inode->i_size) {
393 map->m_llen = map->m_la + 1 - inode->i_size;
394 map->m_la = inode->i_size;
395 map->m_flags = 0;
396 goto out;
397 }
398
399 err = fill_inode_lazy(inode);
400 if (err)
401 goto out;
402
403 lclusterbits = vi->z_logical_clusterbits;
404 ofs = map->m_la;
405 m.lcn = ofs >> lclusterbits;
406 endoff = ofs & ((1 << lclusterbits) - 1);
407
408 err = vle_load_cluster_from_disk(&m, m.lcn);
409 if (err)
410 goto unmap_out;
411
412 map->m_flags = EROFS_MAP_ZIPPED;
413 end = (m.lcn + 1ULL) << lclusterbits;
414
415 switch (m.type) {
416 case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN:
417 if (endoff >= m.clusterofs)
418 map->m_flags &= ~EROFS_MAP_ZIPPED;
419
420 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD:
421 if (endoff >= m.clusterofs) {
422 map->m_la = (m.lcn << lclusterbits) | m.clusterofs;
423 break;
424 }
425
426 if (!m.lcn) {
427 erofs_err(inode->i_sb,
428 "invalid logical cluster 0 at nid %llu",
429 vi->nid);
430 err = -EFSCORRUPTED;
431 goto unmap_out;
432 }
433 end = (m.lcn << lclusterbits) | m.clusterofs;
434 map->m_flags |= EROFS_MAP_FULL_MAPPED;
435 m.delta[0] = 1;
436
437 case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD:
438
439 err = vle_extent_lookback(&m, m.delta[0]);
440 if (err)
441 goto unmap_out;
442 break;
443 default:
444 erofs_err(inode->i_sb,
445 "unknown type %u @ offset %llu of nid %llu",
446 m.type, ofs, vi->nid);
447 err = -EOPNOTSUPP;
448 goto unmap_out;
449 }
450
451 map->m_llen = end - map->m_la;
452 map->m_plen = 1 << lclusterbits;
453 map->m_pa = blknr_to_addr(m.pblk);
454 map->m_flags |= EROFS_MAP_MAPPED;
455
456 unmap_out:
457 if (m.kaddr)
458 kunmap_atomic(m.kaddr);
459
460 out:
461 erofs_dbg("%s, m_la %llu m_pa %llu m_llen %llu m_plen %llu m_flags 0%o",
462 __func__, map->m_la, map->m_pa,
463 map->m_llen, map->m_plen, map->m_flags);
464
465 trace_z_erofs_map_blocks_iter_exit(inode, map, flags, err);
466
467
468 DBG_BUGON(err < 0 && err != -ENOMEM);
469 return err;
470 }
471