root/drivers/gpu/drm/nouveau/nvkm/subdev/bios/init.c

DEFINITIONS

1. init_exec
2. init_exec_set
3. init_exec_inv
4. init_exec_force
5. init_or
6. init_link
7. init_head
8. init_conn
9. init_nvreg
10. init_rd32
11. init_wr32
12. init_mask
13. init_rdport
14. init_wrport
15. init_rdvgai
16. init_wrvgai
17. init_i2c
18. init_rdi2cr
19. init_wri2cr
20. init_aux
21. init_rdauxr
22. init_wrauxr
23. init_prog_pll
24. init_table
25. init_table_
26. init_script
27. init_unknown_script
28. init_ram_restrict_group_count
29. init_ram_restrict
30. init_xlat_
31. init_condition_met
32. init_io_condition_met
33. init_io_flag_condition_met
34. init_shift
35. init_tmds_reg
36. init_reserved
37. init_done
38. init_io_restrict_prog
39. init_repeat
40. init_io_restrict_pll
41. init_end_repeat
42. init_copy
43. init_not
44. init_io_flag_condition
45. init_generic_condition
46. init_io_mask_or
47. init_io_or
48. init_andn_reg
49. init_or_reg
50. init_idx_addr_latched
51. init_io_restrict_pll2
52. init_pll2
53. init_i2c_byte
54. init_zm_i2c_byte
55. init_zm_i2c
56. init_tmds
57. init_zm_tmds_group
58. init_cr_idx_adr_latch
59. init_cr
60. init_zm_cr
61. init_zm_cr_group
62. init_condition_time
63. init_ltime
64. init_zm_reg_sequence
65. init_pll_indirect
66. init_zm_reg_indirect
67. init_sub_direct
68. init_jump
69. init_i2c_if
70. init_copy_nv_reg
71. init_zm_index_io
72. init_compute_mem
73. init_reset
74. init_configure_mem_clk
75. init_configure_mem
76. init_configure_clk
77. init_configure_preinit
78. init_io
79. init_sub
80. init_ram_condition
81. init_nv_reg
82. init_macro
83. init_resume
84. init_strap_condition
85. init_time
86. init_condition
87. init_io_condition
88. init_zm_reg16
89. init_index_io
90. init_pll
91. init_zm_reg
92. init_ram_restrict_pll
93. init_reset_begun
94. init_reset_end
95. init_gpio
96. init_ram_restrict_zm_reg_group
97. init_copy_zm_reg
98. init_zm_reg_group
99. init_xlat
100. init_zm_mask_add
101. init_auxch
102. init_zm_auxch
103. init_i2c_long_if
104. init_gpio_ne
105. nvbios_exec
106. nvbios_post

   1 /*
   2  * Copyright 2012 Red Hat Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Ben Skeggs
  23  */
  24 #include <subdev/bios.h>
  25 #include <subdev/bios/bit.h>
  26 #include <subdev/bios/bmp.h>
  27 #include <subdev/bios/conn.h>
  28 #include <subdev/bios/dcb.h>
  29 #include <subdev/bios/dp.h>
  30 #include <subdev/bios/gpio.h>
  31 #include <subdev/bios/init.h>
  32 #include <subdev/bios/ramcfg.h>
  33 
  34 #include <subdev/devinit.h>
  35 #include <subdev/gpio.h>
  36 #include <subdev/i2c.h>
  37 #include <subdev/vga.h>
  38 
  39 #include <linux/kernel.h>
  40 
  41 #define bioslog(lvl, fmt, args...) do {                                        \
  42         nvkm_printk(init->subdev, lvl, info, "0x%08x[%c]: "fmt,                \
  43                     init->offset, init_exec(init) ?                            \
  44                     '0' + (init->nested - 1) : ' ', ##args);                   \
  45 } while(0)
  46 #define cont(fmt, args...) do {                                                \
  47         if (init->subdev->debug >= NV_DBG_TRACE)                               \
  48                 printk(fmt, ##args);                                           \
  49 } while(0)
  50 #define trace(fmt, args...) bioslog(TRACE, fmt, ##args)
  51 #define warn(fmt, args...) bioslog(WARN, fmt, ##args)
  52 #define error(fmt, args...) bioslog(ERROR, fmt, ##args)
  53 
  54 /******************************************************************************
  55  * init parser control flow helpers
  56  *****************************************************************************/
  57 
  58 static inline bool
  59 init_exec(struct nvbios_init *init)
  60 {
  61         return (init->execute == 1) || ((init->execute & 5) == 5);
  62 }
  63 
  64 static inline void
  65 init_exec_set(struct nvbios_init *init, bool exec)
  66 {
  67         if (exec) init->execute &= 0xfd;
  68         else      init->execute |= 0x02;
  69 }
  70 
  71 static inline void
  72 init_exec_inv(struct nvbios_init *init)
  73 {
  74         init->execute ^= 0x02;
  75 }
  76 
  77 static inline void
  78 init_exec_force(struct nvbios_init *init, bool exec)
  79 {
  80         if (exec) init->execute |= 0x04;
  81         else      init->execute &= 0xfb;
  82 }
  83 
  84 /******************************************************************************
  85  * init parser wrappers for normal register/i2c/whatever accessors
  86  *****************************************************************************/
  87 
  88 static inline int
  89 init_or(struct nvbios_init *init)
  90 {
  91         if (init_exec(init)) {
  92                 if (init->or >= 0)
  93                         return init->or;
  94                 error("script needs OR!!\n");
  95         }
  96         return 0;
  97 }
  98 
  99 static inline int
 100 init_link(struct nvbios_init *init)
 101 {
 102         if (init_exec(init)) {
 103                 if (init->link)
 104                         return init->link == 2;
 105                 error("script needs OR link\n");
 106         }
 107         return 0;
 108 }
 109 
 110 static inline int
 111 init_head(struct nvbios_init *init)
 112 {
 113         if (init_exec(init)) {
 114                 if (init->head >= 0)
 115                         return init->head;
 116                 error("script needs head\n");
 117         }
 118         return 0;
 119 }
 120 
 121 static u8
 122 init_conn(struct nvbios_init *init)
 123 {
 124         struct nvkm_bios *bios = init->subdev->device->bios;
 125         struct nvbios_connE connE;
 126         u8  ver, hdr;
 127         u32 conn;
 128 
 129         if (init_exec(init)) {
 130                 if (init->outp) {
 131                         conn = init->outp->connector;
 132                         conn = nvbios_connEp(bios, conn, &ver, &hdr, &connE);
 133                         if (conn)
 134                                 return connE.type;
 135                 }
 136 
 137                 error("script needs connector type\n");
 138         }
 139 
 140         return 0xff;
 141 }
 142 
 143 static inline u32
 144 init_nvreg(struct nvbios_init *init, u32 reg)
 145 {
 146         struct nvkm_devinit *devinit = init->subdev->device->devinit;
 147 
 148         /* C51 (at least) sometimes has the lower bits set which the VBIOS
 149          * interprets to mean that access needs to go through certain IO
 150          * ports instead.  The NVIDIA binary driver has been seen to access
 151          * these through the NV register address, so lets assume we can
 152          * do the same
 153          */
 154         reg &= ~0x00000003;
 155 
 156         /* GF8+ display scripts need register addresses mangled a bit to
 157          * select a specific CRTC/OR
 158          */
 159         if (init->subdev->device->card_type >= NV_50) {
 160                 if (reg & 0x80000000) {
 161                         reg += init_head(init) * 0x800;
 162                         reg &= ~0x80000000;
 163                 }
 164 
 165                 if (reg & 0x40000000) {
 166                         reg += init_or(init) * 0x800;
 167                         reg &= ~0x40000000;
 168                         if (reg & 0x20000000) {
 169                                 reg += init_link(init) * 0x80;
 170                                 reg &= ~0x20000000;
 171                         }
 172                 }
 173         }
 174 
 175         if (reg & ~0x00fffffc)
 176                 warn("unknown bits in register 0x%08x\n", reg);
 177 
 178         return nvkm_devinit_mmio(devinit, reg);
 179 }
 180 
 181 static u32
 182 init_rd32(struct nvbios_init *init, u32 reg)
 183 {
 184         struct nvkm_device *device = init->subdev->device;
 185         reg = init_nvreg(init, reg);
 186         if (reg != ~0 && init_exec(init))
 187                 return nvkm_rd32(device, reg);
 188         return 0x00000000;
 189 }
 190 
 191 static void
 192 init_wr32(struct nvbios_init *init, u32 reg, u32 val)
 193 {
 194         struct nvkm_device *device = init->subdev->device;
 195         reg = init_nvreg(init, reg);
 196         if (reg != ~0 && init_exec(init))
 197                 nvkm_wr32(device, reg, val);
 198 }
 199 
 200 static u32
 201 init_mask(struct nvbios_init *init, u32 reg, u32 mask, u32 val)
 202 {
 203         struct nvkm_device *device = init->subdev->device;
 204         reg = init_nvreg(init, reg);
 205         if (reg != ~0 && init_exec(init)) {
 206                 u32 tmp = nvkm_rd32(device, reg);
 207                 nvkm_wr32(device, reg, (tmp & ~mask) | val);
 208                 return tmp;
 209         }
 210         return 0x00000000;
 211 }
 212 
 213 static u8
 214 init_rdport(struct nvbios_init *init, u16 port)
 215 {
 216         if (init_exec(init))
 217                 return nvkm_rdport(init->subdev->device, init->head, port);
 218         return 0x00;
 219 }
 220 
 221 static void
 222 init_wrport(struct nvbios_init *init, u16 port, u8 value)
 223 {
 224         if (init_exec(init))
 225                 nvkm_wrport(init->subdev->device, init->head, port, value);
 226 }
 227 
 228 static u8
 229 init_rdvgai(struct nvbios_init *init, u16 port, u8 index)
 230 {
 231         struct nvkm_subdev *subdev = init->subdev;
 232         if (init_exec(init)) {
 233                 int head = init->head < 0 ? 0 : init->head;
 234                 return nvkm_rdvgai(subdev->device, head, port, index);
 235         }
 236         return 0x00;
 237 }
 238 
 239 static void
 240 init_wrvgai(struct nvbios_init *init, u16 port, u8 index, u8 value)
 241 {
 242         struct nvkm_device *device = init->subdev->device;
 243 
 244         /* force head 0 for updates to cr44, it only exists on first head */
 245         if (device->card_type < NV_50) {
 246                 if (port == 0x03d4 && index == 0x44)
 247                         init->head = 0;
 248         }
 249 
 250         if (init_exec(init)) {
 251                 int head = init->head < 0 ? 0 : init->head;
 252                 nvkm_wrvgai(device, head, port, index, value);
 253         }
 254 
 255         /* select head 1 if cr44 write selected it */
 256         if (device->card_type < NV_50) {
 257                 if (port == 0x03d4 && index == 0x44 && value == 3)
 258                         init->head = 1;
 259         }
 260 }
 261 
 262 static struct i2c_adapter *
 263 init_i2c(struct nvbios_init *init, int index)
 264 {
 265         struct nvkm_i2c *i2c = init->subdev->device->i2c;
 266         struct nvkm_i2c_bus *bus;
 267 
 268         if (index == 0xff) {
 269                 index = NVKM_I2C_BUS_PRI;
 270                 if (init->outp && init->outp->i2c_upper_default)
 271                         index = NVKM_I2C_BUS_SEC;
 272         } else
 273         if (index == 0x80) {
 274                 index = NVKM_I2C_BUS_PRI;
 275         } else
 276         if (index == 0x81) {
 277                 index = NVKM_I2C_BUS_SEC;
 278         }
 279 
 280         bus = nvkm_i2c_bus_find(i2c, index);
 281         return bus ? &bus->i2c : NULL;
 282 }
 283 
 284 static int
 285 init_rdi2cr(struct nvbios_init *init, u8 index, u8 addr, u8 reg)
 286 {
 287         struct i2c_adapter *adap = init_i2c(init, index);
 288         if (adap && init_exec(init))
 289                 return nvkm_rdi2cr(adap, addr, reg);
 290         return -ENODEV;
 291 }
 292 
 293 static int
 294 init_wri2cr(struct nvbios_init *init, u8 index, u8 addr, u8 reg, u8 val)
 295 {
 296         struct i2c_adapter *adap = init_i2c(init, index);
 297         if (adap && init_exec(init))
 298                 return nvkm_wri2cr(adap, addr, reg, val);
 299         return -ENODEV;
 300 }
 301 
 302 static struct nvkm_i2c_aux *
 303 init_aux(struct nvbios_init *init)
 304 {
 305         struct nvkm_i2c *i2c = init->subdev->device->i2c;
 306         if (!init->outp) {
 307                 if (init_exec(init))
 308                         error("script needs output for aux\n");
 309                 return NULL;
 310         }
 311         return nvkm_i2c_aux_find(i2c, init->outp->i2c_index);
 312 }
 313 
 314 static u8
 315 init_rdauxr(struct nvbios_init *init, u32 addr)
 316 {
 317         struct nvkm_i2c_aux *aux = init_aux(init);
 318         u8 data;
 319 
 320         if (aux && init_exec(init)) {
 321                 int ret = nvkm_rdaux(aux, addr, &data, 1);
 322                 if (ret == 0)
 323                         return data;
 324                 trace("auxch read failed with %d\n", ret);
 325         }
 326 
 327         return 0x00;
 328 }
 329 
 330 static int
 331 init_wrauxr(struct nvbios_init *init, u32 addr, u8 data)
 332 {
 333         struct nvkm_i2c_aux *aux = init_aux(init);
 334         if (aux && init_exec(init)) {
 335                 int ret = nvkm_wraux(aux, addr, &data, 1);
 336                 if (ret)
 337                         trace("auxch write failed with %d\n", ret);
 338                 return ret;
 339         }
 340         return -ENODEV;
 341 }
 342 
 343 static void
 344 init_prog_pll(struct nvbios_init *init, u32 id, u32 freq)
 345 {
 346         struct nvkm_devinit *devinit = init->subdev->device->devinit;
 347         if (init_exec(init)) {
 348                 int ret = nvkm_devinit_pll_set(devinit, id, freq);
 349                 if (ret)
 350                         warn("failed to prog pll 0x%08x to %dkHz\n", id, freq);
 351         }
 352 }
 353 
 354 /******************************************************************************
 355  * parsing of bios structures that are required to execute init tables
 356  *****************************************************************************/
 357 
 358 static u16
 359 init_table(struct nvkm_bios *bios, u16 *len)
 360 {
 361         struct bit_entry bit_I;
 362 
 363         if (!bit_entry(bios, 'I', &bit_I)) {
 364                 *len = bit_I.length;
 365                 return bit_I.offset;
 366         }
 367 
 368         if (bmp_version(bios) >= 0x0510) {
 369                 *len = 14;
 370                 return bios->bmp_offset + 75;
 371         }
 372 
 373         return 0x0000;
 374 }
 375 
 376 static u16
 377 init_table_(struct nvbios_init *init, u16 offset, const char *name)
 378 {
 379         struct nvkm_bios *bios = init->subdev->device->bios;
 380         u16 len, data = init_table(bios, &len);
 381         if (data) {
 382                 if (len >= offset + 2) {
 383                         data = nvbios_rd16(bios, data + offset);
 384                         if (data)
 385                                 return data;
 386 
 387                         warn("%s pointer invalid\n", name);
 388                         return 0x0000;
 389                 }
 390 
 391                 warn("init data too short for %s pointer", name);
 392                 return 0x0000;
 393         }
 394 
 395         warn("init data not found\n");
 396         return 0x0000;
 397 }
 398 
 399 #define init_script_table(b) init_table_((b), 0x00, "script table")
 400 #define init_macro_index_table(b) init_table_((b), 0x02, "macro index table")
 401 #define init_macro_table(b) init_table_((b), 0x04, "macro table")
 402 #define init_condition_table(b) init_table_((b), 0x06, "condition table")
 403 #define init_io_condition_table(b) init_table_((b), 0x08, "io condition table")
 404 #define init_io_flag_condition_table(b) init_table_((b), 0x0a, "io flag conditon table")
 405 #define init_function_table(b) init_table_((b), 0x0c, "function table")
 406 #define init_xlat_table(b) init_table_((b), 0x10, "xlat table");
 407 
 408 static u16
 409 init_script(struct nvkm_bios *bios, int index)
 410 {
 411         struct nvbios_init init = { .subdev = &bios->subdev };
 412         u16 bmp_ver = bmp_version(bios), data;
 413 
 414         if (bmp_ver && bmp_ver < 0x0510) {
 415                 if (index > 1 || bmp_ver < 0x0100)
 416                         return 0x0000;
 417 
 418                 data = bios->bmp_offset + (bmp_ver < 0x0200 ? 14 : 18);
 419                 return nvbios_rd16(bios, data + (index * 2));
 420         }
 421 
 422         data = init_script_table(&init);
 423         if (data)
 424                 return nvbios_rd16(bios, data + (index * 2));
 425 
 426         return 0x0000;
 427 }
 428 
 429 static u16
 430 init_unknown_script(struct nvkm_bios *bios)
 431 {
 432         u16 len, data = init_table(bios, &len);
 433         if (data && len >= 16)
 434                 return nvbios_rd16(bios, data + 14);
 435         return 0x0000;
 436 }
 437 
 438 static u8
 439 init_ram_restrict_group_count(struct nvbios_init *init)
 440 {
 441         return nvbios_ramcfg_count(init->subdev->device->bios);
 442 }
 443 
 444 static u8
 445 init_ram_restrict(struct nvbios_init *init)
 446 {
 447         /* This appears to be the behaviour of the VBIOS parser, and *is*
 448          * important to cache the NV_PEXTDEV_BOOT0 on later chipsets to
 449          * avoid fucking up the memory controller (somehow) by reading it
 450          * on every INIT_RAM_RESTRICT_ZM_GROUP opcode.
 451          *
 452          * Preserving the non-caching behaviour on earlier chipsets just
 453          * in case *not* re-reading the strap causes similar breakage.
 454          */
 455         if (!init->ramcfg || init->subdev->device->bios->version.major < 0x70)
 456                 init->ramcfg = 0x80000000 | nvbios_ramcfg_index(init->subdev);
 457         return (init->ramcfg & 0x7fffffff);
 458 }
 459 
 460 static u8
 461 init_xlat_(struct nvbios_init *init, u8 index, u8 offset)
 462 {
 463         struct nvkm_bios *bios = init->subdev->device->bios;
 464         u16 table = init_xlat_table(init);
 465         if (table) {
 466                 u16 data = nvbios_rd16(bios, table + (index * 2));
 467                 if (data)
 468                         return nvbios_rd08(bios, data + offset);
 469                 warn("xlat table pointer %d invalid\n", index);
 470         }
 471         return 0x00;
 472 }
 473 
 474 /******************************************************************************
 475  * utility functions used by various init opcode handlers
 476  *****************************************************************************/
 477 
 478 static bool
 479 init_condition_met(struct nvbios_init *init, u8 cond)
 480 {
 481         struct nvkm_bios *bios = init->subdev->device->bios;
 482         u16 table = init_condition_table(init);
 483         if (table) {
 484                 u32 reg = nvbios_rd32(bios, table + (cond * 12) + 0);
 485                 u32 msk = nvbios_rd32(bios, table + (cond * 12) + 4);
 486                 u32 val = nvbios_rd32(bios, table + (cond * 12) + 8);
 487                 trace("\t[0x%02x] (R[0x%06x] & 0x%08x) == 0x%08x\n",
 488                       cond, reg, msk, val);
 489                 return (init_rd32(init, reg) & msk) == val;
 490         }
 491         return false;
 492 }
 493 
 494 static bool
 495 init_io_condition_met(struct nvbios_init *init, u8 cond)
 496 {
 497         struct nvkm_bios *bios = init->subdev->device->bios;
 498         u16 table = init_io_condition_table(init);
 499         if (table) {
 500                 u16 port = nvbios_rd16(bios, table + (cond * 5) + 0);
 501                 u8 index = nvbios_rd08(bios, table + (cond * 5) + 2);
 502                 u8  mask = nvbios_rd08(bios, table + (cond * 5) + 3);
 503                 u8 value = nvbios_rd08(bios, table + (cond * 5) + 4);
 504                 trace("\t[0x%02x] (0x%04x[0x%02x] & 0x%02x) == 0x%02x\n",
 505                       cond, port, index, mask, value);
 506                 return (init_rdvgai(init, port, index) & mask) == value;
 507         }
 508         return false;
 509 }
 510 
 511 static bool
 512 init_io_flag_condition_met(struct nvbios_init *init, u8 cond)
 513 {
 514         struct nvkm_bios *bios = init->subdev->device->bios;
 515         u16 table = init_io_flag_condition_table(init);
 516         if (table) {
 517                 u16 port = nvbios_rd16(bios, table + (cond * 9) + 0);
 518                 u8 index = nvbios_rd08(bios, table + (cond * 9) + 2);
 519                 u8  mask = nvbios_rd08(bios, table + (cond * 9) + 3);
 520                 u8 shift = nvbios_rd08(bios, table + (cond * 9) + 4);
 521                 u16 data = nvbios_rd16(bios, table + (cond * 9) + 5);
 522                 u8 dmask = nvbios_rd08(bios, table + (cond * 9) + 7);
 523                 u8 value = nvbios_rd08(bios, table + (cond * 9) + 8);
 524                 u8 ioval = (init_rdvgai(init, port, index) & mask) >> shift;
 525                 return (nvbios_rd08(bios, data + ioval) & dmask) == value;
 526         }
 527         return false;
 528 }
 529 
 530 static inline u32
 531 init_shift(u32 data, u8 shift)
 532 {
 533         if (shift < 0x80)
 534                 return data >> shift;
 535         return data << (0x100 - shift);
 536 }
 537 
 538 static u32
 539 init_tmds_reg(struct nvbios_init *init, u8 tmds)
 540 {
 541         /* For mlv < 0x80, it is an index into a table of TMDS base addresses.
 542          * For mlv == 0x80 use the "or" value of the dcb_entry indexed by
 543          * CR58 for CR57 = 0 to index a table of offsets to the basic
 544          * 0x6808b0 address.
 545          * For mlv == 0x81 use the "or" value of the dcb_entry indexed by
 546          * CR58 for CR57 = 0 to index a table of offsets to the basic
 547          * 0x6808b0 address, and then flip the offset by 8.
 548          */
 549         const int pramdac_offset[13] = {
 550                 0, 0, 0x8, 0, 0x2000, 0, 0, 0, 0x2008, 0, 0, 0, 0x2000 };
 551         const u32 pramdac_table[4] = {
 552                 0x6808b0, 0x6808b8, 0x6828b0, 0x6828b8 };
 553 
 554         if (tmds >= 0x80) {
 555                 if (init->outp) {
 556                         u32 dacoffset = pramdac_offset[init->outp->or];
 557                         if (tmds == 0x81)
 558                                 dacoffset ^= 8;
 559                         return 0x6808b0 + dacoffset;
 560                 }
 561 
 562                 if (init_exec(init))
 563                         error("tmds opcodes need dcb\n");
 564         } else {
 565                 if (tmds < ARRAY_SIZE(pramdac_table))
 566                         return pramdac_table[tmds];
 567 
 568                 error("tmds selector 0x%02x unknown\n", tmds);
 569         }
 570 
 571         return 0;
 572 }
 573 
 574 /******************************************************************************
 575  * init opcode handlers
 576  *****************************************************************************/
 577 
 578 /**
 579  * init_reserved - stub for various unknown/unused single-byte opcodes
 580  *
 581  */
 582 static void
 583 init_reserved(struct nvbios_init *init)
 584 {
 585         struct nvkm_bios *bios = init->subdev->device->bios;
 586         u8 opcode = nvbios_rd08(bios, init->offset);
 587         u8 length, i;
 588 
 589         switch (opcode) {
 590         case 0xaa:
 591                 length = 4;
 592                 break;
 593         default:
 594                 length = 1;
 595                 break;
 596         }
 597 
 598         trace("RESERVED 0x%02x\t", opcode);
 599         for (i = 1; i < length; i++)
 600                 cont(" 0x%02x", nvbios_rd08(bios, init->offset + i));
 601         cont("\n");
 602         init->offset += length;
 603 }
 604 
 605 /**
 606  * INIT_DONE - opcode 0x71
 607  *
 608  */
 609 static void
 610 init_done(struct nvbios_init *init)
 611 {
 612         trace("DONE\n");
 613         init->offset = 0x0000;
 614 }
 615 
 616 /**
 617  * INIT_IO_RESTRICT_PROG - opcode 0x32
 618  *
 619  */
 620 static void
 621 init_io_restrict_prog(struct nvbios_init *init)
 622 {
 623         struct nvkm_bios *bios = init->subdev->device->bios;
 624         u16 port = nvbios_rd16(bios, init->offset + 1);
 625         u8 index = nvbios_rd08(bios, init->offset + 3);
 626         u8  mask = nvbios_rd08(bios, init->offset + 4);
 627         u8 shift = nvbios_rd08(bios, init->offset + 5);
 628         u8 count = nvbios_rd08(bios, init->offset + 6);
 629         u32  reg = nvbios_rd32(bios, init->offset + 7);
 630         u8 conf, i;
 631 
 632         trace("IO_RESTRICT_PROG\tR[0x%06x] = "
 633               "((0x%04x[0x%02x] & 0x%02x) >> %d) [{\n",
 634               reg, port, index, mask, shift);
 635         init->offset += 11;
 636 
 637         conf = (init_rdvgai(init, port, index) & mask) >> shift;
 638         for (i = 0; i < count; i++) {
 639                 u32 data = nvbios_rd32(bios, init->offset);
 640 
 641                 if (i == conf) {
 642                         trace("\t0x%08x *\n", data);
 643                         init_wr32(init, reg, data);
 644                 } else {
 645                         trace("\t0x%08x\n", data);
 646                 }
 647 
 648                 init->offset += 4;
 649         }
 650         trace("}]\n");
 651 }
 652 
 653 /**
 654  * INIT_REPEAT - opcode 0x33
 655  *
 656  */
 657 static void
 658 init_repeat(struct nvbios_init *init)
 659 {
 660         struct nvkm_bios *bios = init->subdev->device->bios;
 661         u8 count = nvbios_rd08(bios, init->offset + 1);
 662         u16 repeat = init->repeat;
 663 
 664         trace("REPEAT\t0x%02x\n", count);
 665         init->offset += 2;
 666 
 667         init->repeat = init->offset;
 668         init->repend = init->offset;
 669         while (count--) {
 670                 init->offset = init->repeat;
 671                 nvbios_exec(init);
 672                 if (count)
 673                         trace("REPEAT\t0x%02x\n", count);
 674         }
 675         init->offset = init->repend;
 676         init->repeat = repeat;
 677 }
 678 
 679 /**
 680  * INIT_IO_RESTRICT_PLL - opcode 0x34
 681  *
 682  */
 683 static void
 684 init_io_restrict_pll(struct nvbios_init *init)
 685 {
 686         struct nvkm_bios *bios = init->subdev->device->bios;
 687         u16 port = nvbios_rd16(bios, init->offset + 1);
 688         u8 index = nvbios_rd08(bios, init->offset + 3);
 689         u8  mask = nvbios_rd08(bios, init->offset + 4);
 690         u8 shift = nvbios_rd08(bios, init->offset + 5);
 691         s8  iofc = nvbios_rd08(bios, init->offset + 6);
 692         u8 count = nvbios_rd08(bios, init->offset + 7);
 693         u32  reg = nvbios_rd32(bios, init->offset + 8);
 694         u8 conf, i;
 695 
 696         trace("IO_RESTRICT_PLL\tR[0x%06x] =PLL= "
 697               "((0x%04x[0x%02x] & 0x%02x) >> 0x%02x) IOFCOND 0x%02x [{\n",
 698               reg, port, index, mask, shift, iofc);
 699         init->offset += 12;
 700 
 701         conf = (init_rdvgai(init, port, index) & mask) >> shift;
 702         for (i = 0; i < count; i++) {
 703                 u32 freq = nvbios_rd16(bios, init->offset) * 10;
 704 
 705                 if (i == conf) {
 706                         trace("\t%dkHz *\n", freq);
 707                         if (iofc > 0 && init_io_flag_condition_met(init, iofc))
 708                                 freq *= 2;
 709                         init_prog_pll(init, reg, freq);
 710                 } else {
 711                         trace("\t%dkHz\n", freq);
 712                 }
 713 
 714                 init->offset += 2;
 715         }
 716         trace("}]\n");
 717 }
 718 
 719 /**
 720  * INIT_END_REPEAT - opcode 0x36
 721  *
 722  */
 723 static void
 724 init_end_repeat(struct nvbios_init *init)
 725 {
 726         trace("END_REPEAT\n");
 727         init->offset += 1;
 728 
 729         if (init->repeat) {
 730                 init->repend = init->offset;
 731                 init->offset = 0;
 732         }
 733 }
 734 
 735 /**
 736  * INIT_COPY - opcode 0x37
 737  *
 738  */
 739 static void
 740 init_copy(struct nvbios_init *init)
 741 {
 742         struct nvkm_bios *bios = init->subdev->device->bios;
 743         u32  reg = nvbios_rd32(bios, init->offset + 1);
 744         u8 shift = nvbios_rd08(bios, init->offset + 5);
 745         u8 smask = nvbios_rd08(bios, init->offset + 6);
 746         u16 port = nvbios_rd16(bios, init->offset + 7);
 747         u8 index = nvbios_rd08(bios, init->offset + 9);
 748         u8  mask = nvbios_rd08(bios, init->offset + 10);
 749         u8  data;
 750 
 751         trace("COPY\t0x%04x[0x%02x] &= 0x%02x |= "
 752               "((R[0x%06x] %s 0x%02x) & 0x%02x)\n",
 753               port, index, mask, reg, (shift & 0x80) ? "<<" : ">>",
 754               (shift & 0x80) ? (0x100 - shift) : shift, smask);
 755         init->offset += 11;
 756 
 757         data  = init_rdvgai(init, port, index) & mask;
 758         data |= init_shift(init_rd32(init, reg), shift) & smask;
 759         init_wrvgai(init, port, index, data);
 760 }
 761 
 762 /**
 763  * INIT_NOT - opcode 0x38
 764  *
 765  */
 766 static void
 767 init_not(struct nvbios_init *init)
 768 {
 769         trace("NOT\n");
 770         init->offset += 1;
 771         init_exec_inv(init);
 772 }
 773 
 774 /**
 775  * INIT_IO_FLAG_CONDITION - opcode 0x39
 776  *
 777  */
 778 static void
 779 init_io_flag_condition(struct nvbios_init *init)
 780 {
 781         struct nvkm_bios *bios = init->subdev->device->bios;
 782         u8 cond = nvbios_rd08(bios, init->offset + 1);
 783 
 784         trace("IO_FLAG_CONDITION\t0x%02x\n", cond);
 785         init->offset += 2;
 786 
 787         if (!init_io_flag_condition_met(init, cond))
 788                 init_exec_set(init, false);
 789 }
 790 
 791 /**
 792  * INIT_GENERIC_CONDITION - opcode 0x3a
 793  *
 794  */
 795 static void
 796 init_generic_condition(struct nvbios_init *init)
 797 {
 798         struct nvkm_bios *bios = init->subdev->device->bios;
 799         struct nvbios_dpout info;
 800         u8  cond = nvbios_rd08(bios, init->offset + 1);
 801         u8  size = nvbios_rd08(bios, init->offset + 2);
 802         u8  ver, hdr, cnt, len;
 803         u16 data;
 804 
 805         trace("GENERIC_CONDITION\t0x%02x 0x%02x\n", cond, size);
 806         init->offset += 3;
 807 
 808         switch (cond) {
 809         case 0: /* CONDITION_ID_INT_DP. */
 810                 if (init_conn(init) != DCB_CONNECTOR_eDP)
 811                         init_exec_set(init, false);
 812                 break;
 813         case 1: /* CONDITION_ID_USE_SPPLL0. */
 814         case 2: /* CONDITION_ID_USE_SPPLL1. */
 815                 if ( init->outp &&
 816                     (data = nvbios_dpout_match(bios, DCB_OUTPUT_DP,
 817                                                (init->outp->or << 0) |
 818                                                (init->outp->sorconf.link << 6),
 819                                                &ver, &hdr, &cnt, &len, &info)))
 820                 {
 821                         if (!(info.flags & cond))
 822                                 init_exec_set(init, false);
 823                         break;
 824                 }
 825 
 826                 if (init_exec(init))
 827                         warn("script needs dp output table data\n");
 828                 break;
 829         case 5: /* CONDITION_ID_ASSR_SUPPORT. */
 830                 if (!(init_rdauxr(init, 0x0d) & 1))
 831                         init_exec_set(init, false);
 832                 break;
 833         case 7: /* CONDITION_ID_NO_PANEL_SEQ_DELAYS. */
 834                 init_exec_set(init, false);
 835                 break;
 836         default:
 837                 warn("INIT_GENERIC_CONDITION: unknown 0x%02x\n", cond);
 838                 init->offset += size;
 839                 break;
 840         }
 841 }
 842 
 843 /**
 844  * INIT_IO_MASK_OR - opcode 0x3b
 845  *
 846  */
 847 static void
 848 init_io_mask_or(struct nvbios_init *init)
 849 {
 850         struct nvkm_bios *bios = init->subdev->device->bios;
 851         u8 index = nvbios_rd08(bios, init->offset + 1);
 852         u8    or = init_or(init);
 853         u8  data;
 854 
 855         trace("IO_MASK_OR\t0x03d4[0x%02x] &= ~(1 << 0x%02x)\n", index, or);
 856         init->offset += 2;
 857 
 858         data = init_rdvgai(init, 0x03d4, index);
 859         init_wrvgai(init, 0x03d4, index, data &= ~(1 << or));
 860 }
 861 
 862 /**
 863  * INIT_IO_OR - opcode 0x3c
 864  *
 865  */
 866 static void
 867 init_io_or(struct nvbios_init *init)
 868 {
 869         struct nvkm_bios *bios = init->subdev->device->bios;
 870         u8 index = nvbios_rd08(bios, init->offset + 1);
 871         u8    or = init_or(init);
 872         u8  data;
 873 
 874         trace("IO_OR\t0x03d4[0x%02x] |= (1 << 0x%02x)\n", index, or);
 875         init->offset += 2;
 876 
 877         data = init_rdvgai(init, 0x03d4, index);
 878         init_wrvgai(init, 0x03d4, index, data | (1 << or));
 879 }
 880 
 881 /**
 882  * INIT_ANDN_REG - opcode 0x47
 883  *
 884  */
 885 static void
 886 init_andn_reg(struct nvbios_init *init)
 887 {
 888         struct nvkm_bios *bios = init->subdev->device->bios;
 889         u32  reg = nvbios_rd32(bios, init->offset + 1);
 890         u32 mask = nvbios_rd32(bios, init->offset + 5);
 891 
 892         trace("ANDN_REG\tR[0x%06x] &= ~0x%08x\n", reg, mask);
 893         init->offset += 9;
 894 
 895         init_mask(init, reg, mask, 0);
 896 }
 897 
 898 /**
 899  * INIT_OR_REG - opcode 0x48
 900  *
 901  */
 902 static void
 903 init_or_reg(struct nvbios_init *init)
 904 {
 905         struct nvkm_bios *bios = init->subdev->device->bios;
 906         u32  reg = nvbios_rd32(bios, init->offset + 1);
 907         u32 mask = nvbios_rd32(bios, init->offset + 5);
 908 
 909         trace("OR_REG\tR[0x%06x] |= 0x%08x\n", reg, mask);
 910         init->offset += 9;
 911 
 912         init_mask(init, reg, 0, mask);
 913 }
 914 
 915 /**
 916  * INIT_INDEX_ADDRESS_LATCHED - opcode 0x49
 917  *
 918  */
 919 static void
 920 init_idx_addr_latched(struct nvbios_init *init)
 921 {
 922         struct nvkm_bios *bios = init->subdev->device->bios;
 923         u32 creg = nvbios_rd32(bios, init->offset + 1);
 924         u32 dreg = nvbios_rd32(bios, init->offset + 5);
 925         u32 mask = nvbios_rd32(bios, init->offset + 9);
 926         u32 data = nvbios_rd32(bios, init->offset + 13);
 927         u8 count = nvbios_rd08(bios, init->offset + 17);
 928 
 929         trace("INDEX_ADDRESS_LATCHED\tR[0x%06x] : R[0x%06x]\n", creg, dreg);
 930         trace("\tCTRL &= 0x%08x |= 0x%08x\n", mask, data);
 931         init->offset += 18;
 932 
 933         while (count--) {
 934                 u8 iaddr = nvbios_rd08(bios, init->offset + 0);
 935                 u8 idata = nvbios_rd08(bios, init->offset + 1);
 936 
 937                 trace("\t[0x%02x] = 0x%02x\n", iaddr, idata);
 938                 init->offset += 2;
 939 
 940                 init_wr32(init, dreg, idata);
 941                 init_mask(init, creg, ~mask, data | iaddr);
 942         }
 943 }
 944 
 945 /**
 946  * INIT_IO_RESTRICT_PLL2 - opcode 0x4a
 947  *
 948  */
 949 static void
 950 init_io_restrict_pll2(struct nvbios_init *init)
 951 {
 952         struct nvkm_bios *bios = init->subdev->device->bios;
 953         u16 port = nvbios_rd16(bios, init->offset + 1);
 954         u8 index = nvbios_rd08(bios, init->offset + 3);
 955         u8  mask = nvbios_rd08(bios, init->offset + 4);
 956         u8 shift = nvbios_rd08(bios, init->offset + 5);
 957         u8 count = nvbios_rd08(bios, init->offset + 6);
 958         u32  reg = nvbios_rd32(bios, init->offset + 7);
 959         u8  conf, i;
 960 
 961         trace("IO_RESTRICT_PLL2\t"
 962               "R[0x%06x] =PLL= ((0x%04x[0x%02x] & 0x%02x) >> 0x%02x) [{\n",
 963               reg, port, index, mask, shift);
 964         init->offset += 11;
 965 
 966         conf = (init_rdvgai(init, port, index) & mask) >> shift;
 967         for (i = 0; i < count; i++) {
 968                 u32 freq = nvbios_rd32(bios, init->offset);
 969                 if (i == conf) {
 970                         trace("\t%dkHz *\n", freq);
 971                         init_prog_pll(init, reg, freq);
 972                 } else {
 973                         trace("\t%dkHz\n", freq);
 974                 }
 975                 init->offset += 4;
 976         }
 977         trace("}]\n");
 978 }
 979 
 980 /**
 981  * INIT_PLL2 - opcode 0x4b
 982  *
 983  */
 984 static void
 985 init_pll2(struct nvbios_init *init)
 986 {
 987         struct nvkm_bios *bios = init->subdev->device->bios;
 988         u32  reg = nvbios_rd32(bios, init->offset + 1);
 989         u32 freq = nvbios_rd32(bios, init->offset + 5);
 990 
 991         trace("PLL2\tR[0x%06x] =PLL= %dkHz\n", reg, freq);
 992         init->offset += 9;
 993 
 994         init_prog_pll(init, reg, freq);
 995 }
 996 
 997 /**
 998  * INIT_I2C_BYTE - opcode 0x4c
 999  *
1000  */
1001 static void
1002 init_i2c_byte(struct nvbios_init *init)
1003 {
1004         struct nvkm_bios *bios = init->subdev->device->bios;
1005         u8 index = nvbios_rd08(bios, init->offset + 1);
1006         u8  addr = nvbios_rd08(bios, init->offset + 2) >> 1;
1007         u8 count = nvbios_rd08(bios, init->offset + 3);
1008 
1009         trace("I2C_BYTE\tI2C[0x%02x][0x%02x]\n", index, addr);
1010         init->offset += 4;
1011 
1012         while (count--) {
1013                 u8  reg = nvbios_rd08(bios, init->offset + 0);
1014                 u8 mask = nvbios_rd08(bios, init->offset + 1);
1015                 u8 data = nvbios_rd08(bios, init->offset + 2);
1016                 int val;
1017 
1018                 trace("\t[0x%02x] &= 0x%02x |= 0x%02x\n", reg, mask, data);
1019                 init->offset += 3;
1020 
1021                 val = init_rdi2cr(init, index, addr, reg);
1022                 if (val < 0)
1023                         continue;
1024                 init_wri2cr(init, index, addr, reg, (val & mask) | data);
1025         }
1026 }
1027 
1028 /**
1029  * INIT_ZM_I2C_BYTE - opcode 0x4d
1030  *
1031  */
1032 static void
1033 init_zm_i2c_byte(struct nvbios_init *init)
1034 {
1035         struct nvkm_bios *bios = init->subdev->device->bios;
1036         u8 index = nvbios_rd08(bios, init->offset + 1);
1037         u8  addr = nvbios_rd08(bios, init->offset + 2) >> 1;
1038         u8 count = nvbios_rd08(bios, init->offset + 3);
1039 
1040         trace("ZM_I2C_BYTE\tI2C[0x%02x][0x%02x]\n", index, addr);
1041         init->offset += 4;
1042 
1043         while (count--) {
1044                 u8  reg = nvbios_rd08(bios, init->offset + 0);
1045                 u8 data = nvbios_rd08(bios, init->offset + 1);
1046 
1047                 trace("\t[0x%02x] = 0x%02x\n", reg, data);
1048                 init->offset += 2;
1049 
1050                 init_wri2cr(init, index, addr, reg, data);
1051         }
1052 }
1053 
1054 /**
1055  * INIT_ZM_I2C - opcode 0x4e
1056  *
1057  */
1058 static void
1059 init_zm_i2c(struct nvbios_init *init)
1060 {
1061         struct nvkm_bios *bios = init->subdev->device->bios;
1062         u8 index = nvbios_rd08(bios, init->offset + 1);
1063         u8  addr = nvbios_rd08(bios, init->offset + 2) >> 1;
1064         u8 count = nvbios_rd08(bios, init->offset + 3);
1065         u8 data[256], i;
1066 
1067         trace("ZM_I2C\tI2C[0x%02x][0x%02x]\n", index, addr);
1068         init->offset += 4;
1069 
1070         for (i = 0; i < count; i++) {
1071                 data[i] = nvbios_rd08(bios, init->offset);
1072                 trace("\t0x%02x\n", data[i]);
1073                 init->offset++;
1074         }
1075 
1076         if (init_exec(init)) {
1077                 struct i2c_adapter *adap = init_i2c(init, index);
1078                 struct i2c_msg msg = {
1079                         .addr = addr, .flags = 0, .len = count, .buf = data,
1080                 };
1081                 int ret;
1082 
1083                 if (adap && (ret = i2c_transfer(adap, &msg, 1)) != 1)
1084                         warn("i2c wr failed, %d\n", ret);
1085         }
1086 }
1087 
1088 /**
1089  * INIT_TMDS - opcode 0x4f
1090  *
1091  */
1092 static void
1093 init_tmds(struct nvbios_init *init)
1094 {
1095         struct nvkm_bios *bios = init->subdev->device->bios;
1096         u8 tmds = nvbios_rd08(bios, init->offset + 1);
1097         u8 addr = nvbios_rd08(bios, init->offset + 2);
1098         u8 mask = nvbios_rd08(bios, init->offset + 3);
1099         u8 data = nvbios_rd08(bios, init->offset + 4);
1100         u32 reg = init_tmds_reg(init, tmds);
1101 
1102         trace("TMDS\tT[0x%02x][0x%02x] &= 0x%02x |= 0x%02x\n",
1103               tmds, addr, mask, data);
1104         init->offset += 5;
1105 
1106         if (reg == 0)
1107                 return;
1108 
1109         init_wr32(init, reg + 0, addr | 0x00010000);
1110         init_wr32(init, reg + 4, data | (init_rd32(init, reg + 4) & mask));
1111         init_wr32(init, reg + 0, addr);
1112 }
1113 
1114 /**
1115  * INIT_ZM_TMDS_GROUP - opcode 0x50
1116  *
1117  */
1118 static void
1119 init_zm_tmds_group(struct nvbios_init *init)
1120 {
1121         struct nvkm_bios *bios = init->subdev->device->bios;
1122         u8  tmds = nvbios_rd08(bios, init->offset + 1);
1123         u8 count = nvbios_rd08(bios, init->offset + 2);
1124         u32  reg = init_tmds_reg(init, tmds);
1125 
1126         trace("TMDS_ZM_GROUP\tT[0x%02x]\n", tmds);
1127         init->offset += 3;
1128 
1129         while (count--) {
1130                 u8 addr = nvbios_rd08(bios, init->offset + 0);
1131                 u8 data = nvbios_rd08(bios, init->offset + 1);
1132 
1133                 trace("\t[0x%02x] = 0x%02x\n", addr, data);
1134                 init->offset += 2;
1135 
1136                 init_wr32(init, reg + 4, data);
1137                 init_wr32(init, reg + 0, addr);
1138         }
1139 }
1140 
1141 /**
1142  * INIT_CR_INDEX_ADDRESS_LATCHED - opcode 0x51
1143  *
1144  */
1145 static void
1146 init_cr_idx_adr_latch(struct nvbios_init *init)
1147 {
1148         struct nvkm_bios *bios = init->subdev->device->bios;
1149         u8 addr0 = nvbios_rd08(bios, init->offset + 1);
1150         u8 addr1 = nvbios_rd08(bios, init->offset + 2);
1151         u8  base = nvbios_rd08(bios, init->offset + 3);
1152         u8 count = nvbios_rd08(bios, init->offset + 4);
1153         u8 save0;
1154 
1155         trace("CR_INDEX_ADDR C[%02x] C[%02x]\n", addr0, addr1);
1156         init->offset += 5;
1157 
1158         save0 = init_rdvgai(init, 0x03d4, addr0);
1159         while (count--) {
1160                 u8 data = nvbios_rd08(bios, init->offset);
1161 
1162                 trace("\t\t[0x%02x] = 0x%02x\n", base, data);
1163                 init->offset += 1;
1164 
1165                 init_wrvgai(init, 0x03d4, addr0, base++);
1166                 init_wrvgai(init, 0x03d4, addr1, data);
1167         }
1168         init_wrvgai(init, 0x03d4, addr0, save0);
1169 }
1170 
1171 /**
1172  * INIT_CR - opcode 0x52
1173  *
1174  */
1175 static void
1176 init_cr(struct nvbios_init *init)
1177 {
1178         struct nvkm_bios *bios = init->subdev->device->bios;
1179         u8 addr = nvbios_rd08(bios, init->offset + 1);
1180         u8 mask = nvbios_rd08(bios, init->offset + 2);
1181         u8 data = nvbios_rd08(bios, init->offset + 3);
1182         u8 val;
1183 
1184         trace("CR\t\tC[0x%02x] &= 0x%02x |= 0x%02x\n", addr, mask, data);
1185         init->offset += 4;
1186 
1187         val = init_rdvgai(init, 0x03d4, addr) & mask;
1188         init_wrvgai(init, 0x03d4, addr, val | data);
1189 }
1190 
1191 /**
1192  * INIT_ZM_CR - opcode 0x53
1193  *
1194  */
1195 static void
1196 init_zm_cr(struct nvbios_init *init)
1197 {
1198         struct nvkm_bios *bios = init->subdev->device->bios;
1199         u8 addr = nvbios_rd08(bios, init->offset + 1);
1200         u8 data = nvbios_rd08(bios, init->offset + 2);
1201 
1202         trace("ZM_CR\tC[0x%02x] = 0x%02x\n", addr,  data);
1203         init->offset += 3;
1204 
1205         init_wrvgai(init, 0x03d4, addr, data);
1206 }
1207 
1208 /**
1209  * INIT_ZM_CR_GROUP - opcode 0x54
1210  *
1211  */
1212 static void
1213 init_zm_cr_group(struct nvbios_init *init)
1214 {
1215         struct nvkm_bios *bios = init->subdev->device->bios;
1216         u8 count = nvbios_rd08(bios, init->offset + 1);
1217 
1218         trace("ZM_CR_GROUP\n");
1219         init->offset += 2;
1220 
1221         while (count--) {
1222                 u8 addr = nvbios_rd08(bios, init->offset + 0);
1223                 u8 data = nvbios_rd08(bios, init->offset + 1);
1224 
1225                 trace("\t\tC[0x%02x] = 0x%02x\n", addr, data);
1226                 init->offset += 2;
1227 
1228                 init_wrvgai(init, 0x03d4, addr, data);
1229         }
1230 }
1231 
1232 /**
1233  * INIT_CONDITION_TIME - opcode 0x56
1234  *
1235  */
1236 static void
1237 init_condition_time(struct nvbios_init *init)
1238 {
1239         struct nvkm_bios *bios = init->subdev->device->bios;
1240         u8  cond = nvbios_rd08(bios, init->offset + 1);
1241         u8 retry = nvbios_rd08(bios, init->offset + 2);
1242         u8  wait = min((u16)retry * 50, 100);
1243 
1244         trace("CONDITION_TIME\t0x%02x 0x%02x\n", cond, retry);
1245         init->offset += 3;
1246 
1247         if (!init_exec(init))
1248                 return;
1249 
1250         while (wait--) {
1251                 if (init_condition_met(init, cond))
1252                         return;
1253                 mdelay(20);
1254         }
1255 
1256         init_exec_set(init, false);
1257 }
1258 
1259 /**
1260  * INIT_LTIME - opcode 0x57
1261  *
1262  */
1263 static void
1264 init_ltime(struct nvbios_init *init)
1265 {
1266         struct nvkm_bios *bios = init->subdev->device->bios;
1267         u16 msec = nvbios_rd16(bios, init->offset + 1);
1268 
1269         trace("LTIME\t0x%04x\n", msec);
1270         init->offset += 3;
1271 
1272         if (init_exec(init))
1273                 mdelay(msec);
1274 }
1275 
1276 /**
1277  * INIT_ZM_REG_SEQUENCE - opcode 0x58
1278  *
1279  */
1280 static void
1281 init_zm_reg_sequence(struct nvbios_init *init)
1282 {
1283         struct nvkm_bios *bios = init->subdev->device->bios;
1284         u32 base = nvbios_rd32(bios, init->offset + 1);
1285         u8 count = nvbios_rd08(bios, init->offset + 5);
1286 
1287         trace("ZM_REG_SEQUENCE\t0x%02x\n", count);
1288         init->offset += 6;
1289 
1290         while (count--) {
1291                 u32 data = nvbios_rd32(bios, init->offset);
1292 
1293                 trace("\t\tR[0x%06x] = 0x%08x\n", base, data);
1294                 init->offset += 4;
1295 
1296                 init_wr32(init, base, data);
1297                 base += 4;
1298         }
1299 }
1300 
1301 /**
1302  * INIT_PLL_INDIRECT - opcode 0x59
1303  *
1304  */
1305 static void
1306 init_pll_indirect(struct nvbios_init *init)
1307 {
1308         struct nvkm_bios *bios = init->subdev->device->bios;
1309         u32  reg = nvbios_rd32(bios, init->offset + 1);
1310         u16 addr = nvbios_rd16(bios, init->offset + 5);
1311         u32 freq = (u32)nvbios_rd16(bios, addr) * 1000;
1312 
1313         trace("PLL_INDIRECT\tR[0x%06x] =PLL= VBIOS[%04x] = %dkHz\n",
1314               reg, addr, freq);
1315         init->offset += 7;
1316 
1317         init_prog_pll(init, reg, freq);
1318 }
1319 
1320 /**
1321  * INIT_ZM_REG_INDIRECT - opcode 0x5a
1322  *
1323  */
1324 static void
1325 init_zm_reg_indirect(struct nvbios_init *init)
1326 {
1327         struct nvkm_bios *bios = init->subdev->device->bios;
1328         u32  reg = nvbios_rd32(bios, init->offset + 1);
1329         u16 addr = nvbios_rd16(bios, init->offset + 5);
1330         u32 data = nvbios_rd32(bios, addr);
1331 
1332         trace("ZM_REG_INDIRECT\tR[0x%06x] = VBIOS[0x%04x] = 0x%08x\n",
1333               reg, addr, data);
1334         init->offset += 7;
1335 
1336         init_wr32(init, addr, data);
1337 }
1338 
1339 /**
1340  * INIT_SUB_DIRECT - opcode 0x5b
1341  *
1342  */
1343 static void
1344 init_sub_direct(struct nvbios_init *init)
1345 {
1346         struct nvkm_bios *bios = init->subdev->device->bios;
1347         u16 addr = nvbios_rd16(bios, init->offset + 1);
1348         u16 save;
1349 
1350         trace("SUB_DIRECT\t0x%04x\n", addr);
1351 
1352         if (init_exec(init)) {
1353                 save = init->offset;
1354                 init->offset = addr;
1355                 if (nvbios_exec(init)) {
1356                         error("error parsing sub-table\n");
1357                         return;
1358                 }
1359                 init->offset = save;
1360         }
1361 
1362         init->offset += 3;
1363 }
1364 
1365 /**
1366  * INIT_JUMP - opcode 0x5c
1367  *
1368  */
1369 static void
1370 init_jump(struct nvbios_init *init)
1371 {
1372         struct nvkm_bios *bios = init->subdev->device->bios;
1373         u16 offset = nvbios_rd16(bios, init->offset + 1);
1374 
1375         trace("JUMP\t0x%04x\n", offset);
1376 
1377         if (init_exec(init))
1378                 init->offset = offset;
1379         else
1380                 init->offset += 3;
1381 }
1382 
1383 /**
1384  * INIT_I2C_IF - opcode 0x5e
1385  *
1386  */
1387 static void
1388 init_i2c_if(struct nvbios_init *init)
1389 {
1390         struct nvkm_bios *bios = init->subdev->device->bios;
1391         u8 index = nvbios_rd08(bios, init->offset + 1);
1392         u8  addr = nvbios_rd08(bios, init->offset + 2);
1393         u8   reg = nvbios_rd08(bios, init->offset + 3);
1394         u8  mask = nvbios_rd08(bios, init->offset + 4);
1395         u8  data = nvbios_rd08(bios, init->offset + 5);
1396         u8 value;
1397 
1398         trace("I2C_IF\tI2C[0x%02x][0x%02x][0x%02x] & 0x%02x == 0x%02x\n",
1399               index, addr, reg, mask, data);
1400         init->offset += 6;
1401         init_exec_force(init, true);
1402 
1403         value = init_rdi2cr(init, index, addr, reg);
1404         if ((value & mask) != data)
1405                 init_exec_set(init, false);
1406 
1407         init_exec_force(init, false);
1408 }
1409 
1410 /**
1411  * INIT_COPY_NV_REG - opcode 0x5f
1412  *
1413  */
1414 static void
1415 init_copy_nv_reg(struct nvbios_init *init)
1416 {
1417         struct nvkm_bios *bios = init->subdev->device->bios;
1418         u32  sreg = nvbios_rd32(bios, init->offset + 1);
1419         u8  shift = nvbios_rd08(bios, init->offset + 5);
1420         u32 smask = nvbios_rd32(bios, init->offset + 6);
1421         u32  sxor = nvbios_rd32(bios, init->offset + 10);
1422         u32  dreg = nvbios_rd32(bios, init->offset + 14);
1423         u32 dmask = nvbios_rd32(bios, init->offset + 18);
1424         u32 data;
1425 
1426         trace("COPY_NV_REG\tR[0x%06x] &= 0x%08x |= "
1427               "((R[0x%06x] %s 0x%02x) & 0x%08x ^ 0x%08x)\n",
1428               dreg, dmask, sreg, (shift & 0x80) ? "<<" : ">>",
1429               (shift & 0x80) ? (0x100 - shift) : shift, smask, sxor);
1430         init->offset += 22;
1431 
1432         data = init_shift(init_rd32(init, sreg), shift);
1433         init_mask(init, dreg, ~dmask, (data & smask) ^ sxor);
1434 }
1435 
1436 /**
1437  * INIT_ZM_INDEX_IO - opcode 0x62
1438  *
1439  */
1440 static void
1441 init_zm_index_io(struct nvbios_init *init)
1442 {
1443         struct nvkm_bios *bios = init->subdev->device->bios;
1444         u16 port = nvbios_rd16(bios, init->offset + 1);
1445         u8 index = nvbios_rd08(bios, init->offset + 3);
1446         u8  data = nvbios_rd08(bios, init->offset + 4);
1447 
1448         trace("ZM_INDEX_IO\tI[0x%04x][0x%02x] = 0x%02x\n", port, index, data);
1449         init->offset += 5;
1450 
1451         init_wrvgai(init, port, index, data);
1452 }
1453 
1454 /**
1455  * INIT_COMPUTE_MEM - opcode 0x63
1456  *
1457  */
1458 static void
1459 init_compute_mem(struct nvbios_init *init)
1460 {
1461         struct nvkm_devinit *devinit = init->subdev->device->devinit;
1462 
1463         trace("COMPUTE_MEM\n");
1464         init->offset += 1;
1465 
1466         init_exec_force(init, true);
1467         if (init_exec(init))
1468                 nvkm_devinit_meminit(devinit);
1469         init_exec_force(init, false);
1470 }
1471 
1472 /**
1473  * INIT_RESET - opcode 0x65
1474  *
1475  */
1476 static void
1477 init_reset(struct nvbios_init *init)
1478 {
1479         struct nvkm_bios *bios = init->subdev->device->bios;
1480         u32   reg = nvbios_rd32(bios, init->offset + 1);
1481         u32 data1 = nvbios_rd32(bios, init->offset + 5);
1482         u32 data2 = nvbios_rd32(bios, init->offset + 9);
1483         u32 savepci19;
1484 
1485         trace("RESET\tR[0x%08x] = 0x%08x, 0x%08x", reg, data1, data2);
1486         init->offset += 13;
1487         init_exec_force(init, true);
1488 
1489         savepci19 = init_mask(init, 0x00184c, 0x00000f00, 0x00000000);
1490         init_wr32(init, reg, data1);
1491         udelay(10);
1492         init_wr32(init, reg, data2);
1493         init_wr32(init, 0x00184c, savepci19);
1494         init_mask(init, 0x001850, 0x00000001, 0x00000000);
1495 
1496         init_exec_force(init, false);
1497 }
1498 
1499 /**
1500  * INIT_CONFIGURE_MEM - opcode 0x66
1501  *
1502  */
1503 static u16
1504 init_configure_mem_clk(struct nvbios_init *init)
1505 {
1506         u16 mdata = bmp_mem_init_table(init->subdev->device->bios);
1507         if (mdata)
1508                 mdata += (init_rdvgai(init, 0x03d4, 0x3c) >> 4) * 66;
1509         return mdata;
1510 }
1511 
1512 static void
1513 init_configure_mem(struct nvbios_init *init)
1514 {
1515         struct nvkm_bios *bios = init->subdev->device->bios;
1516         u16 mdata, sdata;
1517         u32 addr, data;
1518 
1519         trace("CONFIGURE_MEM\n");
1520         init->offset += 1;
1521 
1522         if (bios->version.major > 2) {
1523                 init_done(init);
1524                 return;
1525         }
1526         init_exec_force(init, true);
1527 
1528         mdata = init_configure_mem_clk(init);
1529         sdata = bmp_sdr_seq_table(bios);
1530         if (nvbios_rd08(bios, mdata) & 0x01)
1531                 sdata = bmp_ddr_seq_table(bios);
1532         mdata += 6; /* skip to data */
1533 
1534         data = init_rdvgai(init, 0x03c4, 0x01);
1535         init_wrvgai(init, 0x03c4, 0x01, data | 0x20);
1536 
1537         for (; (addr = nvbios_rd32(bios, sdata)) != 0xffffffff; sdata += 4) {
1538                 switch (addr) {
1539                 case 0x10021c: /* CKE_NORMAL */
1540                 case 0x1002d0: /* CMD_REFRESH */
1541                 case 0x1002d4: /* CMD_PRECHARGE */
1542                         data = 0x00000001;
1543                         break;
1544                 default:
1545                         data = nvbios_rd32(bios, mdata);
1546                         mdata += 4;
1547                         if (data == 0xffffffff)
1548                                 continue;
1549                         break;
1550                 }
1551 
1552                 init_wr32(init, addr, data);
1553         }
1554 
1555         init_exec_force(init, false);
1556 }
1557 
1558 /**
1559  * INIT_CONFIGURE_CLK - opcode 0x67
1560  *
1561  */
1562 static void
1563 init_configure_clk(struct nvbios_init *init)
1564 {
1565         struct nvkm_bios *bios = init->subdev->device->bios;
1566         u16 mdata, clock;
1567 
1568         trace("CONFIGURE_CLK\n");
1569         init->offset += 1;
1570 
1571         if (bios->version.major > 2) {
1572                 init_done(init);
1573                 return;
1574         }
1575         init_exec_force(init, true);
1576 
1577         mdata = init_configure_mem_clk(init);
1578 
1579         /* NVPLL */
1580         clock = nvbios_rd16(bios, mdata + 4) * 10;
1581         init_prog_pll(init, 0x680500, clock);
1582 
1583         /* MPLL */
1584         clock = nvbios_rd16(bios, mdata + 2) * 10;
1585         if (nvbios_rd08(bios, mdata) & 0x01)
1586                 clock *= 2;
1587         init_prog_pll(init, 0x680504, clock);
1588 
1589         init_exec_force(init, false);
1590 }
1591 
1592 /**
1593  * INIT_CONFIGURE_PREINIT - opcode 0x68
1594  *
1595  */
1596 static void
1597 init_configure_preinit(struct nvbios_init *init)
1598 {
1599         struct nvkm_bios *bios = init->subdev->device->bios;
1600         u32 strap;
1601 
1602         trace("CONFIGURE_PREINIT\n");
1603         init->offset += 1;
1604 
1605         if (bios->version.major > 2) {
1606                 init_done(init);
1607                 return;
1608         }
1609         init_exec_force(init, true);
1610 
1611         strap = init_rd32(init, 0x101000);
1612         strap = ((strap << 2) & 0xf0) | ((strap & 0x40) >> 6);
1613         init_wrvgai(init, 0x03d4, 0x3c, strap);
1614 
1615         init_exec_force(init, false);
1616 }
1617 
1618 /**
1619  * INIT_IO - opcode 0x69
1620  *
1621  */
1622 static void
1623 init_io(struct nvbios_init *init)
1624 {
1625         struct nvkm_bios *bios = init->subdev->device->bios;
1626         u16 port = nvbios_rd16(bios, init->offset + 1);
1627         u8  mask = nvbios_rd16(bios, init->offset + 3);
1628         u8  data = nvbios_rd16(bios, init->offset + 4);
1629         u8 value;
1630 
1631         trace("IO\t\tI[0x%04x] &= 0x%02x |= 0x%02x\n", port, mask, data);
1632         init->offset += 5;
1633 
1634         /* ummm.. yes.. should really figure out wtf this is and why it's
1635          * needed some day..  it's almost certainly wrong, but, it also
1636          * somehow makes things work...
1637          */
1638         if (bios->subdev.device->card_type >= NV_50 &&
1639             port == 0x03c3 && data == 0x01) {
1640                 init_mask(init, 0x614100, 0xf0800000, 0x00800000);
1641                 init_mask(init, 0x00e18c, 0x00020000, 0x00020000);
1642                 init_mask(init, 0x614900, 0xf0800000, 0x00800000);
1643                 init_mask(init, 0x000200, 0x40000000, 0x00000000);
1644                 mdelay(10);
1645                 init_mask(init, 0x00e18c, 0x00020000, 0x00000000);
1646                 init_mask(init, 0x000200, 0x40000000, 0x40000000);
1647                 init_wr32(init, 0x614100, 0x00800018);
1648                 init_wr32(init, 0x614900, 0x00800018);
1649                 mdelay(10);
1650                 init_wr32(init, 0x614100, 0x10000018);
1651                 init_wr32(init, 0x614900, 0x10000018);
1652         }
1653 
1654         value = init_rdport(init, port) & mask;
1655         init_wrport(init, port, data | value);
1656 }
1657 
1658 /**
1659  * INIT_SUB - opcode 0x6b
1660  *
1661  */
1662 static void
1663 init_sub(struct nvbios_init *init)
1664 {
1665         struct nvkm_bios *bios = init->subdev->device->bios;
1666         u8 index = nvbios_rd08(bios, init->offset + 1);
1667         u16 addr, save;
1668 
1669         trace("SUB\t0x%02x\n", index);
1670 
1671         addr = init_script(bios, index);
1672         if (addr && init_exec(init)) {
1673                 save = init->offset;
1674                 init->offset = addr;
1675                 if (nvbios_exec(init)) {
1676                         error("error parsing sub-table\n");
1677                         return;
1678                 }
1679                 init->offset = save;
1680         }
1681 
1682         init->offset += 2;
1683 }
1684 
1685 /**
1686  * INIT_RAM_CONDITION - opcode 0x6d
1687  *
1688  */
1689 static void
1690 init_ram_condition(struct nvbios_init *init)
1691 {
1692         struct nvkm_bios *bios = init->subdev->device->bios;
1693         u8  mask = nvbios_rd08(bios, init->offset + 1);
1694         u8 value = nvbios_rd08(bios, init->offset + 2);
1695 
1696         trace("RAM_CONDITION\t"
1697               "(R[0x100000] & 0x%02x) == 0x%02x\n", mask, value);
1698         init->offset += 3;
1699 
1700         if ((init_rd32(init, 0x100000) & mask) != value)
1701                 init_exec_set(init, false);
1702 }
1703 
1704 /**
1705  * INIT_NV_REG - opcode 0x6e
1706  *
1707  */
1708 static void
1709 init_nv_reg(struct nvbios_init *init)
1710 {
1711         struct nvkm_bios *bios = init->subdev->device->bios;
1712         u32  reg = nvbios_rd32(bios, init->offset + 1);
1713         u32 mask = nvbios_rd32(bios, init->offset + 5);
1714         u32 data = nvbios_rd32(bios, init->offset + 9);
1715 
1716         trace("NV_REG\tR[0x%06x] &= 0x%08x |= 0x%08x\n", reg, mask, data);
1717         init->offset += 13;
1718 
1719         init_mask(init, reg, ~mask, data);
1720 }
1721 
1722 /**
1723  * INIT_MACRO - opcode 0x6f
1724  *
1725  */
1726 static void
1727 init_macro(struct nvbios_init *init)
1728 {
1729         struct nvkm_bios *bios = init->subdev->device->bios;
1730         u8  macro = nvbios_rd08(bios, init->offset + 1);
1731         u16 table;
1732 
1733         trace("MACRO\t0x%02x\n", macro);
1734 
1735         table = init_macro_table(init);
1736         if (table) {
1737                 u32 addr = nvbios_rd32(bios, table + (macro * 8) + 0);
1738                 u32 data = nvbios_rd32(bios, table + (macro * 8) + 4);
1739                 trace("\t\tR[0x%06x] = 0x%08x\n", addr, data);
1740                 init_wr32(init, addr, data);
1741         }
1742 
1743         init->offset += 2;
1744 }
1745 
1746 /**
1747  * INIT_RESUME - opcode 0x72
1748  *
1749  */
1750 static void
1751 init_resume(struct nvbios_init *init)
1752 {
1753         trace("RESUME\n");
1754         init->offset += 1;
1755         init_exec_set(init, true);
1756 }
1757 
1758 /**
1759  * INIT_STRAP_CONDITION - opcode 0x73
1760  *
1761  */
1762 static void
1763 init_strap_condition(struct nvbios_init *init)
1764 {
1765         struct nvkm_bios *bios = init->subdev->device->bios;
1766         u32 mask = nvbios_rd32(bios, init->offset + 1);
1767         u32 value = nvbios_rd32(bios, init->offset + 5);
1768 
1769         trace("STRAP_CONDITION\t(R[0x101000] & 0x%08x) == 0x%08x\n", mask, value);
1770         init->offset += 9;
1771 
1772         if ((init_rd32(init, 0x101000) & mask) != value)
1773                 init_exec_set(init, false);
1774 }
1775 
1776 /**
1777  * INIT_TIME - opcode 0x74
1778  *
1779  */
1780 static void
1781 init_time(struct nvbios_init *init)
1782 {
1783         struct nvkm_bios *bios = init->subdev->device->bios;
1784         u16 usec = nvbios_rd16(bios, init->offset + 1);
1785 
1786         trace("TIME\t0x%04x\n", usec);
1787         init->offset += 3;
1788 
1789         if (init_exec(init)) {
1790                 if (usec < 1000)
1791                         udelay(usec);
1792                 else
1793                         mdelay((usec + 900) / 1000);
1794         }
1795 }
1796 
1797 /**
1798  * INIT_CONDITION - opcode 0x75
1799  *
1800  */
1801 static void
1802 init_condition(struct nvbios_init *init)
1803 {
1804         struct nvkm_bios *bios = init->subdev->device->bios;
1805         u8 cond = nvbios_rd08(bios, init->offset + 1);
1806 
1807         trace("CONDITION\t0x%02x\n", cond);
1808         init->offset += 2;
1809 
1810         if (!init_condition_met(init, cond))
1811                 init_exec_set(init, false);
1812 }
1813 
1814 /**
1815  * INIT_IO_CONDITION - opcode 0x76
1816  *
1817  */
1818 static void
1819 init_io_condition(struct nvbios_init *init)
1820 {
1821         struct nvkm_bios *bios = init->subdev->device->bios;
1822         u8 cond = nvbios_rd08(bios, init->offset + 1);
1823 
1824         trace("IO_CONDITION\t0x%02x\n", cond);
1825         init->offset += 2;
1826 
1827         if (!init_io_condition_met(init, cond))
1828                 init_exec_set(init, false);
1829 }
1830 
1831 /**
1832  * INIT_ZM_REG16 - opcode 0x77
1833  *
1834  */
1835 static void
1836 init_zm_reg16(struct nvbios_init *init)
1837 {
1838         struct nvkm_bios *bios = init->subdev->device->bios;
1839         u32 addr = nvbios_rd32(bios, init->offset + 1);
1840         u16 data = nvbios_rd16(bios, init->offset + 5);
1841 
1842         trace("ZM_REG\tR[0x%06x] = 0x%04x\n", addr, data);
1843         init->offset += 7;
1844 
1845         init_wr32(init, addr, data);
1846 }
1847 
1848 /**
1849  * INIT_INDEX_IO - opcode 0x78
1850  *
1851  */
1852 static void
1853 init_index_io(struct nvbios_init *init)
1854 {
1855         struct nvkm_bios *bios = init->subdev->device->bios;
1856         u16 port = nvbios_rd16(bios, init->offset + 1);
1857         u8 index = nvbios_rd16(bios, init->offset + 3);
1858         u8  mask = nvbios_rd08(bios, init->offset + 4);
1859         u8  data = nvbios_rd08(bios, init->offset + 5);
1860         u8 value;
1861 
1862         trace("INDEX_IO\tI[0x%04x][0x%02x] &= 0x%02x |= 0x%02x\n",
1863               port, index, mask, data);
1864         init->offset += 6;
1865 
1866         value = init_rdvgai(init, port, index) & mask;
1867         init_wrvgai(init, port, index, data | value);
1868 }
1869 
1870 /**
1871  * INIT_PLL - opcode 0x79
1872  *
1873  */
1874 static void
1875 init_pll(struct nvbios_init *init)
1876 {
1877         struct nvkm_bios *bios = init->subdev->device->bios;
1878         u32  reg = nvbios_rd32(bios, init->offset + 1);
1879         u32 freq = nvbios_rd16(bios, init->offset + 5) * 10;
1880 
1881         trace("PLL\tR[0x%06x] =PLL= %dkHz\n", reg, freq);
1882         init->offset += 7;
1883 
1884         init_prog_pll(init, reg, freq);
1885 }
1886 
1887 /**
1888  * INIT_ZM_REG - opcode 0x7a
1889  *
1890  */
1891 static void
1892 init_zm_reg(struct nvbios_init *init)
1893 {
1894         struct nvkm_bios *bios = init->subdev->device->bios;
1895         u32 addr = nvbios_rd32(bios, init->offset + 1);
1896         u32 data = nvbios_rd32(bios, init->offset + 5);
1897 
1898         trace("ZM_REG\tR[0x%06x] = 0x%08x\n", addr, data);
1899         init->offset += 9;
1900 
1901         if (addr == 0x000200)
1902                 data |= 0x00000001;
1903 
1904         init_wr32(init, addr, data);
1905 }
1906 
1907 /**
1908  * INIT_RAM_RESTRICT_PLL - opcde 0x87
1909  *
1910  */
1911 static void
1912 init_ram_restrict_pll(struct nvbios_init *init)
1913 {
1914         struct nvkm_bios *bios = init->subdev->device->bios;
1915         u8  type = nvbios_rd08(bios, init->offset + 1);
1916         u8 count = init_ram_restrict_group_count(init);
1917         u8 strap = init_ram_restrict(init);
1918         u8 cconf;
1919 
1920         trace("RAM_RESTRICT_PLL\t0x%02x\n", type);
1921         init->offset += 2;
1922 
1923         for (cconf = 0; cconf < count; cconf++) {
1924                 u32 freq = nvbios_rd32(bios, init->offset);
1925 
1926                 if (cconf == strap) {
1927                         trace("%dkHz *\n", freq);
1928                         init_prog_pll(init, type, freq);
1929                 } else {
1930                         trace("%dkHz\n", freq);
1931                 }
1932 
1933                 init->offset += 4;
1934         }
1935 }
1936 
1937 /**
1938  * INIT_RESET_BEGUN - opcode 0x8c
1939  *
1940  */
1941 static void
1942 init_reset_begun(struct nvbios_init *init)
1943 {
1944         trace("RESET_BEGUN\n");
1945         init->offset += 1;
1946 }
1947 
1948 /**
1949  * INIT_RESET_END - opcode 0x8d
1950  *
1951  */
1952 static void
1953 init_reset_end(struct nvbios_init *init)
1954 {
1955         trace("RESET_END\n");
1956         init->offset += 1;
1957 }
1958 
1959 /**
1960  * INIT_GPIO - opcode 0x8e
1961  *
1962  */
1963 static void
1964 init_gpio(struct nvbios_init *init)
1965 {
1966         struct nvkm_gpio *gpio = init->subdev->device->gpio;
1967 
1968         trace("GPIO\n");
1969         init->offset += 1;
1970 
1971         if (init_exec(init))
1972                 nvkm_gpio_reset(gpio, DCB_GPIO_UNUSED);
1973 }
1974 
1975 /**
1976  * INIT_RAM_RESTRICT_ZM_GROUP - opcode 0x8f
1977  *
1978  */
1979 static void
1980 init_ram_restrict_zm_reg_group(struct nvbios_init *init)
1981 {
1982         struct nvkm_bios *bios = init->subdev->device->bios;
1983         u32 addr = nvbios_rd32(bios, init->offset + 1);
1984         u8  incr = nvbios_rd08(bios, init->offset + 5);
1985         u8   num = nvbios_rd08(bios, init->offset + 6);
1986         u8 count = init_ram_restrict_group_count(init);
1987         u8 index = init_ram_restrict(init);
1988         u8 i, j;
1989 
1990         trace("RAM_RESTRICT_ZM_REG_GROUP\t"
1991               "R[0x%08x] 0x%02x 0x%02x\n", addr, incr, num);
1992         init->offset += 7;
1993 
1994         for (i = 0; i < num; i++) {
1995                 trace("\tR[0x%06x] = {\n", addr);
1996                 for (j = 0; j < count; j++) {
1997                         u32 data = nvbios_rd32(bios, init->offset);
1998 
1999                         if (j == index) {
2000                                 trace("\t\t0x%08x *\n", data);
2001                                 init_wr32(init, addr, data);
2002                         } else {
2003                                 trace("\t\t0x%08x\n", data);
2004                         }
2005 
2006                         init->offset += 4;
2007                 }
2008                 trace("\t}\n");
2009                 addr += incr;
2010         }
2011 }
2012 
2013 /**
2014  * INIT_COPY_ZM_REG - opcode 0x90
2015  *
2016  */
2017 static void
2018 init_copy_zm_reg(struct nvbios_init *init)
2019 {
2020         struct nvkm_bios *bios = init->subdev->device->bios;
2021         u32 sreg = nvbios_rd32(bios, init->offset + 1);
2022         u32 dreg = nvbios_rd32(bios, init->offset + 5);
2023 
2024         trace("COPY_ZM_REG\tR[0x%06x] = R[0x%06x]\n", dreg, sreg);
2025         init->offset += 9;
2026 
2027         init_wr32(init, dreg, init_rd32(init, sreg));
2028 }
2029 
2030 /**
2031  * INIT_ZM_REG_GROUP - opcode 0x91
2032  *
2033  */
2034 static void
2035 init_zm_reg_group(struct nvbios_init *init)
2036 {
2037         struct nvkm_bios *bios = init->subdev->device->bios;
2038         u32 addr = nvbios_rd32(bios, init->offset + 1);
2039         u8 count = nvbios_rd08(bios, init->offset + 5);
2040 
2041         trace("ZM_REG_GROUP\tR[0x%06x] =\n", addr);
2042         init->offset += 6;
2043 
2044         while (count--) {
2045                 u32 data = nvbios_rd32(bios, init->offset);
2046                 trace("\t0x%08x\n", data);
2047                 init_wr32(init, addr, data);
2048                 init->offset += 4;
2049         }
2050 }
2051 
2052 /**
2053  * INIT_XLAT - opcode 0x96
2054  *
2055  */
2056 static void
2057 init_xlat(struct nvbios_init *init)
2058 {
2059         struct nvkm_bios *bios = init->subdev->device->bios;
2060         u32 saddr = nvbios_rd32(bios, init->offset + 1);
2061         u8 sshift = nvbios_rd08(bios, init->offset + 5);
2062         u8  smask = nvbios_rd08(bios, init->offset + 6);
2063         u8  index = nvbios_rd08(bios, init->offset + 7);
2064         u32 daddr = nvbios_rd32(bios, init->offset + 8);
2065         u32 dmask = nvbios_rd32(bios, init->offset + 12);
2066         u8  shift = nvbios_rd08(bios, init->offset + 16);
2067         u32 data;
2068 
2069         trace("INIT_XLAT\tR[0x%06x] &= 0x%08x |= "
2070               "(X%02x((R[0x%06x] %s 0x%02x) & 0x%02x) << 0x%02x)\n",
2071               daddr, dmask, index, saddr, (sshift & 0x80) ? "<<" : ">>",
2072               (sshift & 0x80) ? (0x100 - sshift) : sshift, smask, shift);
2073         init->offset += 17;
2074 
2075         data = init_shift(init_rd32(init, saddr), sshift) & smask;
2076         data = init_xlat_(init, index, data) << shift;
2077         init_mask(init, daddr, ~dmask, data);
2078 }
2079 
2080 /**
2081  * INIT_ZM_MASK_ADD - opcode 0x97
2082  *
2083  */
2084 static void
2085 init_zm_mask_add(struct nvbios_init *init)
2086 {
2087         struct nvkm_bios *bios = init->subdev->device->bios;
2088         u32 addr = nvbios_rd32(bios, init->offset + 1);
2089         u32 mask = nvbios_rd32(bios, init->offset + 5);
2090         u32  add = nvbios_rd32(bios, init->offset + 9);
2091         u32 data;
2092 
2093         trace("ZM_MASK_ADD\tR[0x%06x] &= 0x%08x += 0x%08x\n", addr, mask, add);
2094         init->offset += 13;
2095 
2096         data =  init_rd32(init, addr);
2097         data = (data & mask) | ((data + add) & ~mask);
2098         init_wr32(init, addr, data);
2099 }
2100 
2101 /**
2102  * INIT_AUXCH - opcode 0x98
2103  *
2104  */
2105 static void
2106 init_auxch(struct nvbios_init *init)
2107 {
2108         struct nvkm_bios *bios = init->subdev->device->bios;
2109         u32 addr = nvbios_rd32(bios, init->offset + 1);
2110         u8 count = nvbios_rd08(bios, init->offset + 5);
2111 
2112         trace("AUXCH\tAUX[0x%08x] 0x%02x\n", addr, count);
2113         init->offset += 6;
2114 
2115         while (count--) {
2116                 u8 mask = nvbios_rd08(bios, init->offset + 0);
2117                 u8 data = nvbios_rd08(bios, init->offset + 1);
2118                 trace("\tAUX[0x%08x] &= 0x%02x |= 0x%02x\n", addr, mask, data);
2119                 mask = init_rdauxr(init, addr) & mask;
2120                 init_wrauxr(init, addr, mask | data);
2121                 init->offset += 2;
2122         }
2123 }
2124 
2125 /**
2126  * INIT_AUXCH - opcode 0x99
2127  *
2128  */
2129 static void
2130 init_zm_auxch(struct nvbios_init *init)
2131 {
2132         struct nvkm_bios *bios = init->subdev->device->bios;
2133         u32 addr = nvbios_rd32(bios, init->offset + 1);
2134         u8 count = nvbios_rd08(bios, init->offset + 5);
2135 
2136         trace("ZM_AUXCH\tAUX[0x%08x] 0x%02x\n", addr, count);
2137         init->offset += 6;
2138 
2139         while (count--) {
2140                 u8 data = nvbios_rd08(bios, init->offset + 0);
2141                 trace("\tAUX[0x%08x] = 0x%02x\n", addr, data);
2142                 init_wrauxr(init, addr, data);
2143                 init->offset += 1;
2144         }
2145 }
2146 
2147 /**
2148  * INIT_I2C_LONG_IF - opcode 0x9a
2149  *
2150  */
2151 static void
2152 init_i2c_long_if(struct nvbios_init *init)
2153 {
2154         struct nvkm_bios *bios = init->subdev->device->bios;
2155         u8 index = nvbios_rd08(bios, init->offset + 1);
2156         u8  addr = nvbios_rd08(bios, init->offset + 2) >> 1;
2157         u8 reglo = nvbios_rd08(bios, init->offset + 3);
2158         u8 reghi = nvbios_rd08(bios, init->offset + 4);
2159         u8  mask = nvbios_rd08(bios, init->offset + 5);
2160         u8  data = nvbios_rd08(bios, init->offset + 6);
2161         struct i2c_adapter *adap;
2162 
2163         trace("I2C_LONG_IF\t"
2164               "I2C[0x%02x][0x%02x][0x%02x%02x] & 0x%02x == 0x%02x\n",
2165               index, addr, reglo, reghi, mask, data);
2166         init->offset += 7;
2167 
2168         adap = init_i2c(init, index);
2169         if (adap) {
2170                 u8 i[2] = { reghi, reglo };
2171                 u8 o[1] = {};
2172                 struct i2c_msg msg[] = {
2173                         { .addr = addr, .flags = 0, .len = 2, .buf = i },
2174                         { .addr = addr, .flags = I2C_M_RD, .len = 1, .buf = o }
2175                 };
2176                 int ret;
2177 
2178                 ret = i2c_transfer(adap, msg, 2);
2179                 if (ret == 2 && ((o[0] & mask) == data))
2180                         return;
2181         }
2182 
2183         init_exec_set(init, false);
2184 }
2185 
2186 /**
2187  * INIT_GPIO_NE - opcode 0xa9
2188  *
2189  */
2190 static void
2191 init_gpio_ne(struct nvbios_init *init)
2192 {
2193         struct nvkm_bios *bios = init->subdev->device->bios;
2194         struct nvkm_gpio *gpio = bios->subdev.device->gpio;
2195         struct dcb_gpio_func func;
2196         u8 count = nvbios_rd08(bios, init->offset + 1);
2197         u8 idx = 0, ver, len;
2198         u16 data, i;
2199 
2200         trace("GPIO_NE\t");
2201         init->offset += 2;
2202 
2203         for (i = init->offset; i < init->offset + count; i++)
2204                 cont("0x%02x ", nvbios_rd08(bios, i));
2205         cont("\n");
2206 
2207         while ((data = dcb_gpio_parse(bios, 0, idx++, &ver, &len, &func))) {
2208                 if (func.func != DCB_GPIO_UNUSED) {
2209                         for (i = init->offset; i < init->offset + count; i++) {
2210                                 if (func.func == nvbios_rd08(bios, i))
2211                                         break;
2212                         }
2213 
2214                         trace("\tFUNC[0x%02x]", func.func);
2215                         if (i == (init->offset + count)) {
2216                                 cont(" *");
2217                                 if (init_exec(init))
2218                                         nvkm_gpio_reset(gpio, func.func);
2219                         }
2220                         cont("\n");
2221                 }
2222         }
2223 
2224         init->offset += count;
2225 }
2226 
2227 static struct nvbios_init_opcode {
2228         void (*exec)(struct nvbios_init *);
2229 } init_opcode[] = {
2230         [0x32] = { init_io_restrict_prog },
2231         [0x33] = { init_repeat },
2232         [0x34] = { init_io_restrict_pll },
2233         [0x36] = { init_end_repeat },
2234         [0x37] = { init_copy },
2235         [0x38] = { init_not },
2236         [0x39] = { init_io_flag_condition },
2237         [0x3a] = { init_generic_condition },
2238         [0x3b] = { init_io_mask_or },
2239         [0x3c] = { init_io_or },
2240         [0x47] = { init_andn_reg },
2241         [0x48] = { init_or_reg },
2242         [0x49] = { init_idx_addr_latched },
2243         [0x4a] = { init_io_restrict_pll2 },
2244         [0x4b] = { init_pll2 },
2245         [0x4c] = { init_i2c_byte },
2246         [0x4d] = { init_zm_i2c_byte },
2247         [0x4e] = { init_zm_i2c },
2248         [0x4f] = { init_tmds },
2249         [0x50] = { init_zm_tmds_group },
2250         [0x51] = { init_cr_idx_adr_latch },
2251         [0x52] = { init_cr },
2252         [0x53] = { init_zm_cr },
2253         [0x54] = { init_zm_cr_group },
2254         [0x56] = { init_condition_time },
2255         [0x57] = { init_ltime },
2256         [0x58] = { init_zm_reg_sequence },
2257         [0x59] = { init_pll_indirect },
2258         [0x5a] = { init_zm_reg_indirect },
2259         [0x5b] = { init_sub_direct },
2260         [0x5c] = { init_jump },
2261         [0x5e] = { init_i2c_if },
2262         [0x5f] = { init_copy_nv_reg },
2263         [0x62] = { init_zm_index_io },
2264         [0x63] = { init_compute_mem },
2265         [0x65] = { init_reset },
2266         [0x66] = { init_configure_mem },
2267         [0x67] = { init_configure_clk },
2268         [0x68] = { init_configure_preinit },
2269         [0x69] = { init_io },
2270         [0x6b] = { init_sub },
2271         [0x6d] = { init_ram_condition },
2272         [0x6e] = { init_nv_reg },
2273         [0x6f] = { init_macro },
2274         [0x71] = { init_done },
2275         [0x72] = { init_resume },
2276         [0x73] = { init_strap_condition },
2277         [0x74] = { init_time },
2278         [0x75] = { init_condition },
2279         [0x76] = { init_io_condition },
2280         [0x77] = { init_zm_reg16 },
2281         [0x78] = { init_index_io },
2282         [0x79] = { init_pll },
2283         [0x7a] = { init_zm_reg },
2284         [0x87] = { init_ram_restrict_pll },
2285         [0x8c] = { init_reset_begun },
2286         [0x8d] = { init_reset_end },
2287         [0x8e] = { init_gpio },
2288         [0x8f] = { init_ram_restrict_zm_reg_group },
2289         [0x90] = { init_copy_zm_reg },
2290         [0x91] = { init_zm_reg_group },
2291         [0x92] = { init_reserved },
2292         [0x96] = { init_xlat },
2293         [0x97] = { init_zm_mask_add },
2294         [0x98] = { init_auxch },
2295         [0x99] = { init_zm_auxch },
2296         [0x9a] = { init_i2c_long_if },
2297         [0xa9] = { init_gpio_ne },
2298         [0xaa] = { init_reserved },
2299 };
2300 
2301 int
2302 nvbios_exec(struct nvbios_init *init)
2303 {
2304         struct nvkm_bios *bios = init->subdev->device->bios;
2305 
2306         init->nested++;
2307         while (init->offset) {
2308                 u8 opcode = nvbios_rd08(bios, init->offset);
2309                 if (opcode >= ARRAY_SIZE(init_opcode) ||
2310                     !init_opcode[opcode].exec) {
2311                         error("unknown opcode 0x%02x\n", opcode);
2312                         return -EINVAL;
2313                 }
2314 
2315                 init_opcode[opcode].exec(init);
2316         }
2317         init->nested--;
2318         return 0;
2319 }
2320 
2321 int
2322 nvbios_post(struct nvkm_subdev *subdev, bool execute)
2323 {
2324         struct nvkm_bios *bios = subdev->device->bios;
2325         int ret = 0;
2326         int i = -1;
2327         u16 data;
2328 
2329         if (execute)
2330                 nvkm_debug(subdev, "running init tables\n");
2331         while (!ret && (data = (init_script(bios, ++i)))) {
2332                 ret = nvbios_init(subdev, data,
2333                         init.execute = execute ? 1 : 0;
2334                       );
2335         }
2336 
2337         /* the vbios parser will run this right after the normal init
2338          * tables, whereas the binary driver appears to run it later.
2339          */
2340         if (!ret && (data = init_unknown_script(bios))) {
2341                 ret = nvbios_init(subdev, data,
2342                         init.execute = execute ? 1 : 0;
2343                       );
2344         }
2345 
2346         return ret;
2347 }