1/*
2 * Copyright 2013 Red Hat Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Ben Skeggs
23 */
24#include "nv50.h"
25#include "ramseq.h"
26
27#include <core/device.h>
28#include <core/option.h>
29#include <subdev/bios.h>
30#include <subdev/bios/perf.h>
31#include <subdev/bios/pll.h>
32#include <subdev/bios/timing.h>
33#include <subdev/clk/pll.h>
34
35struct nv50_ramseq {
36	struct hwsq base;
37	struct hwsq_reg r_0x002504;
38	struct hwsq_reg r_0x004008;
39	struct hwsq_reg r_0x00400c;
40	struct hwsq_reg r_0x00c040;
41	struct hwsq_reg r_0x100210;
42	struct hwsq_reg r_0x1002d0;
43	struct hwsq_reg r_0x1002d4;
44	struct hwsq_reg r_0x1002dc;
45	struct hwsq_reg r_0x100da0[8];
46	struct hwsq_reg r_0x100e20;
47	struct hwsq_reg r_0x100e24;
48	struct hwsq_reg r_0x611200;
49	struct hwsq_reg r_timing[9];
50	struct hwsq_reg r_mr[4];
51};
52
53struct nv50_ram {
54	struct nvkm_ram base;
55	struct nv50_ramseq hwsq;
56};
57
58#define QFX5800NVA0 1
59
60static int
61nv50_ram_calc(struct nvkm_fb *pfb, u32 freq)
62{
63	struct nvkm_bios *bios = nvkm_bios(pfb);
64	struct nv50_ram *ram = (void *)pfb->ram;
65	struct nv50_ramseq *hwsq = &ram->hwsq;
66	struct nvbios_perfE perfE;
67	struct nvbios_pll mpll;
68	struct {
69		u32 data;
70		u8  size;
71	} ramcfg, timing;
72	u8  ver, hdr, cnt, len, strap;
73	int N1, M1, N2, M2, P;
74	int ret, i;
75
76	/* lookup closest matching performance table entry for frequency */
77	i = 0;
78	do {
79		ramcfg.data = nvbios_perfEp(bios, i++, &ver, &hdr, &cnt,
80					    &ramcfg.size, &perfE);
81		if (!ramcfg.data || (ver < 0x25 || ver >= 0x40) ||
82		    (ramcfg.size < 2)) {
83			nv_error(pfb, "invalid/missing perftab entry\n");
84			return -EINVAL;
85		}
86	} while (perfE.memory < freq);
87
88	/* locate specific data set for the attached memory */
89	strap = nvbios_ramcfg_index(nv_subdev(pfb));
90	if (strap >= cnt) {
91		nv_error(pfb, "invalid ramcfg strap\n");
92		return -EINVAL;
93	}
94
95	ramcfg.data += hdr + (strap * ramcfg.size);
96
97	/* lookup memory timings, if bios says they're present */
98	strap = nv_ro08(bios, ramcfg.data + 0x01);
99	if (strap != 0xff) {
100		timing.data = nvbios_timingEe(bios, strap, &ver, &hdr,
101					      &cnt, &len);
102		if (!timing.data || ver != 0x10 || hdr < 0x12) {
103			nv_error(pfb, "invalid/missing timing entry "
104				 "%02x %04x %02x %02x\n",
105				 strap, timing.data, ver, hdr);
106			return -EINVAL;
107		}
108	} else {
109		timing.data = 0;
110	}
111
112	ret = ram_init(hwsq, nv_subdev(pfb));
113	if (ret)
114		return ret;
115
116	ram_wait(hwsq, 0x01, 0x00); /* wait for !vblank */
117	ram_wait(hwsq, 0x01, 0x01); /* wait for vblank */
118	ram_wr32(hwsq, 0x611200, 0x00003300);
119	ram_wr32(hwsq, 0x002504, 0x00000001); /* block fifo */
120	ram_nsec(hwsq, 8000);
121	ram_setf(hwsq, 0x10, 0x00); /* disable fb */
122	ram_wait(hwsq, 0x00, 0x01); /* wait for fb disabled */
123
124	ram_wr32(hwsq, 0x1002d4, 0x00000001); /* precharge */
125	ram_wr32(hwsq, 0x1002d0, 0x00000001); /* refresh */
126	ram_wr32(hwsq, 0x1002d0, 0x00000001); /* refresh */
127	ram_wr32(hwsq, 0x100210, 0x00000000); /* disable auto-refresh */
128	ram_wr32(hwsq, 0x1002dc, 0x00000001); /* enable self-refresh */
129
130	ret = nvbios_pll_parse(bios, 0x004008, &mpll);
131	mpll.vco2.max_freq = 0;
132	if (ret == 0) {
133		ret = nv04_pll_calc(nv_subdev(pfb), &mpll, freq,
134				    &N1, &M1, &N2, &M2, &P);
135		if (ret == 0)
136			ret = -EINVAL;
137	}
138
139	if (ret < 0)
140		return ret;
141
142	ram_mask(hwsq, 0x00c040, 0xc000c000, 0x0000c000);
143	ram_mask(hwsq, 0x004008, 0x00000200, 0x00000200);
144	ram_mask(hwsq, 0x00400c, 0x0000ffff, (N1 << 8) | M1);
145	ram_mask(hwsq, 0x004008, 0x81ff0000, 0x80000000 | (mpll.bias_p << 19) |
146					     (P << 22) | (P << 16));
147#if QFX5800NVA0
148	for (i = 0; i < 8; i++)
149		ram_mask(hwsq, 0x100da0[i], 0x00000000, 0x00000000); /*XXX*/
150#endif
151	ram_nsec(hwsq, 96000); /*XXX*/
152	ram_mask(hwsq, 0x004008, 0x00002200, 0x00002000);
153
154	ram_wr32(hwsq, 0x1002dc, 0x00000000); /* disable self-refresh */
155	ram_wr32(hwsq, 0x100210, 0x80000000); /* enable auto-refresh */
156
157	ram_nsec(hwsq, 12000);
158
159	switch (ram->base.type) {
160	case NV_MEM_TYPE_DDR2:
161		ram_nuke(hwsq, mr[0]); /* force update */
162		ram_mask(hwsq, mr[0], 0x000, 0x000);
163		break;
164	case NV_MEM_TYPE_GDDR3:
165		ram_mask(hwsq, mr[2], 0x000, 0x000);
166		ram_nuke(hwsq, mr[0]); /* force update */
167		ram_mask(hwsq, mr[0], 0x000, 0x000);
168		break;
169	default:
170		break;
171	}
172
173	ram_mask(hwsq, timing[3], 0x00000000, 0x00000000); /*XXX*/
174	ram_mask(hwsq, timing[1], 0x00000000, 0x00000000); /*XXX*/
175	ram_mask(hwsq, timing[6], 0x00000000, 0x00000000); /*XXX*/
176	ram_mask(hwsq, timing[7], 0x00000000, 0x00000000); /*XXX*/
177	ram_mask(hwsq, timing[8], 0x00000000, 0x00000000); /*XXX*/
178	ram_mask(hwsq, timing[0], 0x00000000, 0x00000000); /*XXX*/
179	ram_mask(hwsq, timing[2], 0x00000000, 0x00000000); /*XXX*/
180	ram_mask(hwsq, timing[4], 0x00000000, 0x00000000); /*XXX*/
181	ram_mask(hwsq, timing[5], 0x00000000, 0x00000000); /*XXX*/
182
183	ram_mask(hwsq, timing[0], 0x00000000, 0x00000000); /*XXX*/
184
185#if QFX5800NVA0
186	ram_nuke(hwsq, 0x100e24);
187	ram_mask(hwsq, 0x100e24, 0x00000000, 0x00000000);
188	ram_nuke(hwsq, 0x100e20);
189	ram_mask(hwsq, 0x100e20, 0x00000000, 0x00000000);
190#endif
191
192	ram_mask(hwsq, mr[0], 0x100, 0x100);
193	ram_mask(hwsq, mr[0], 0x100, 0x000);
194
195	ram_setf(hwsq, 0x10, 0x01); /* enable fb */
196	ram_wait(hwsq, 0x00, 0x00); /* wait for fb enabled */
197	ram_wr32(hwsq, 0x611200, 0x00003330);
198	ram_wr32(hwsq, 0x002504, 0x00000000); /* un-block fifo */
199	return 0;
200}
201
202static int
203nv50_ram_prog(struct nvkm_fb *pfb)
204{
205	struct nvkm_device *device = nv_device(pfb);
206	struct nv50_ram *ram = (void *)pfb->ram;
207	struct nv50_ramseq *hwsq = &ram->hwsq;
208
209	ram_exec(hwsq, nvkm_boolopt(device->cfgopt, "NvMemExec", true));
210	return 0;
211}
212
213static void
214nv50_ram_tidy(struct nvkm_fb *pfb)
215{
216	struct nv50_ram *ram = (void *)pfb->ram;
217	struct nv50_ramseq *hwsq = &ram->hwsq;
218	ram_exec(hwsq, false);
219}
220
221void
222__nv50_ram_put(struct nvkm_fb *pfb, struct nvkm_mem *mem)
223{
224	struct nvkm_mm_node *this;
225
226	while (!list_empty(&mem->regions)) {
227		this = list_first_entry(&mem->regions, typeof(*this), rl_entry);
228
229		list_del(&this->rl_entry);
230		nvkm_mm_free(&pfb->vram, &this);
231	}
232
233	nvkm_mm_free(&pfb->tags, &mem->tag);
234}
235
236void
237nv50_ram_put(struct nvkm_fb *pfb, struct nvkm_mem **pmem)
238{
239	struct nvkm_mem *mem = *pmem;
240
241	*pmem = NULL;
242	if (unlikely(mem == NULL))
243		return;
244
245	mutex_lock(&pfb->base.mutex);
246	__nv50_ram_put(pfb, mem);
247	mutex_unlock(&pfb->base.mutex);
248
249	kfree(mem);
250}
251
252int
253nv50_ram_get(struct nvkm_fb *pfb, u64 size, u32 align, u32 ncmin,
254	     u32 memtype, struct nvkm_mem **pmem)
255{
256	struct nvkm_mm *heap = &pfb->vram;
257	struct nvkm_mm *tags = &pfb->tags;
258	struct nvkm_mm_node *r;
259	struct nvkm_mem *mem;
260	int comp = (memtype & 0x300) >> 8;
261	int type = (memtype & 0x07f);
262	int back = (memtype & 0x800);
263	int min, max, ret;
264
265	max = (size >> 12);
266	min = ncmin ? (ncmin >> 12) : max;
267	align >>= 12;
268
269	mem = kzalloc(sizeof(*mem), GFP_KERNEL);
270	if (!mem)
271		return -ENOMEM;
272
273	mutex_lock(&pfb->base.mutex);
274	if (comp) {
275		if (align == 16) {
276			int n = (max >> 4) * comp;
277
278			ret = nvkm_mm_head(tags, 0, 1, n, n, 1, &mem->tag);
279			if (ret)
280				mem->tag = NULL;
281		}
282
283		if (unlikely(!mem->tag))
284			comp = 0;
285	}
286
287	INIT_LIST_HEAD(&mem->regions);
288	mem->memtype = (comp << 7) | type;
289	mem->size = max;
290
291	type = nv50_fb_memtype[type];
292	do {
293		if (back)
294			ret = nvkm_mm_tail(heap, 0, type, max, min, align, &r);
295		else
296			ret = nvkm_mm_head(heap, 0, type, max, min, align, &r);
297		if (ret) {
298			mutex_unlock(&pfb->base.mutex);
299			pfb->ram->put(pfb, &mem);
300			return ret;
301		}
302
303		list_add_tail(&r->rl_entry, &mem->regions);
304		max -= r->length;
305	} while (max);
306	mutex_unlock(&pfb->base.mutex);
307
308	r = list_first_entry(&mem->regions, struct nvkm_mm_node, rl_entry);
309	mem->offset = (u64)r->offset << 12;
310	*pmem = mem;
311	return 0;
312}
313
314static u32
315nv50_fb_vram_rblock(struct nvkm_fb *pfb, struct nvkm_ram *ram)
316{
317	int colbits, rowbitsa, rowbitsb, banks;
318	u64 rowsize, predicted;
319	u32 r0, r4, rt, rblock_size;
320
321	r0 = nv_rd32(pfb, 0x100200);
322	r4 = nv_rd32(pfb, 0x100204);
323	rt = nv_rd32(pfb, 0x100250);
324	nv_debug(pfb, "memcfg 0x%08x 0x%08x 0x%08x 0x%08x\n",
325		 r0, r4, rt, nv_rd32(pfb, 0x001540));
326
327	colbits  =  (r4 & 0x0000f000) >> 12;
328	rowbitsa = ((r4 & 0x000f0000) >> 16) + 8;
329	rowbitsb = ((r4 & 0x00f00000) >> 20) + 8;
330	banks    = 1 << (((r4 & 0x03000000) >> 24) + 2);
331
332	rowsize = ram->parts * banks * (1 << colbits) * 8;
333	predicted = rowsize << rowbitsa;
334	if (r0 & 0x00000004)
335		predicted += rowsize << rowbitsb;
336
337	if (predicted != ram->size) {
338		nv_warn(pfb, "memory controller reports %d MiB VRAM\n",
339			(u32)(ram->size >> 20));
340	}
341
342	rblock_size = rowsize;
343	if (rt & 1)
344		rblock_size *= 3;
345
346	nv_debug(pfb, "rblock %d bytes\n", rblock_size);
347	return rblock_size;
348}
349
350int
351nv50_ram_create_(struct nvkm_object *parent, struct nvkm_object *engine,
352		 struct nvkm_oclass *oclass, int length, void **pobject)
353{
354	const u32 rsvd_head = ( 256 * 1024) >> 12; /* vga memory */
355	const u32 rsvd_tail = (1024 * 1024) >> 12; /* vbios etc */
356	struct nvkm_bios *bios = nvkm_bios(parent);
357	struct nvkm_fb *pfb = nvkm_fb(parent);
358	struct nvkm_ram *ram;
359	int ret;
360
361	ret = nvkm_ram_create_(parent, engine, oclass, length, pobject);
362	ram = *pobject;
363	if (ret)
364		return ret;
365
366	ram->size = nv_rd32(pfb, 0x10020c);
367	ram->size = (ram->size & 0xffffff00) | ((ram->size & 0x000000ff) << 32);
368
369	ram->part_mask = (nv_rd32(pfb, 0x001540) & 0x00ff0000) >> 16;
370	ram->parts = hweight8(ram->part_mask);
371
372	switch (nv_rd32(pfb, 0x100714) & 0x00000007) {
373	case 0: ram->type = NV_MEM_TYPE_DDR1; break;
374	case 1:
375		if (nvkm_fb_bios_memtype(bios) == NV_MEM_TYPE_DDR3)
376			ram->type = NV_MEM_TYPE_DDR3;
377		else
378			ram->type = NV_MEM_TYPE_DDR2;
379		break;
380	case 2: ram->type = NV_MEM_TYPE_GDDR3; break;
381	case 3: ram->type = NV_MEM_TYPE_GDDR4; break;
382	case 4: ram->type = NV_MEM_TYPE_GDDR5; break;
383	default:
384		break;
385	}
386
387	ret = nvkm_mm_init(&pfb->vram, rsvd_head, (ram->size >> 12) -
388			   (rsvd_head + rsvd_tail),
389			   nv50_fb_vram_rblock(pfb, ram) >> 12);
390	if (ret)
391		return ret;
392
393	ram->ranks = (nv_rd32(pfb, 0x100200) & 0x4) ? 2 : 1;
394	ram->tags  =  nv_rd32(pfb, 0x100320);
395	ram->get = nv50_ram_get;
396	ram->put = nv50_ram_put;
397	return 0;
398}
399
400static int
401nv50_ram_ctor(struct nvkm_object *parent, struct nvkm_object *engine,
402	      struct nvkm_oclass *oclass, void *data, u32 datasize,
403	      struct nvkm_object **pobject)
404{
405	struct nv50_ram *ram;
406	int ret, i;
407
408	ret = nv50_ram_create(parent, engine, oclass, &ram);
409	*pobject = nv_object(ram);
410	if (ret)
411		return ret;
412
413	switch (ram->base.type) {
414	case NV_MEM_TYPE_DDR2:
415	case NV_MEM_TYPE_GDDR3:
416		ram->base.calc = nv50_ram_calc;
417		ram->base.prog = nv50_ram_prog;
418		ram->base.tidy = nv50_ram_tidy;
419		break;
420	default:
421		nv_warn(ram, "reclocking of this ram type unsupported\n");
422		return 0;
423	}
424
425	ram->hwsq.r_0x002504 = hwsq_reg(0x002504);
426	ram->hwsq.r_0x00c040 = hwsq_reg(0x00c040);
427	ram->hwsq.r_0x004008 = hwsq_reg(0x004008);
428	ram->hwsq.r_0x00400c = hwsq_reg(0x00400c);
429	ram->hwsq.r_0x100210 = hwsq_reg(0x100210);
430	ram->hwsq.r_0x1002d0 = hwsq_reg(0x1002d0);
431	ram->hwsq.r_0x1002d4 = hwsq_reg(0x1002d4);
432	ram->hwsq.r_0x1002dc = hwsq_reg(0x1002dc);
433	for (i = 0; i < 8; i++)
434		ram->hwsq.r_0x100da0[i] = hwsq_reg(0x100da0 + (i * 0x04));
435	ram->hwsq.r_0x100e20 = hwsq_reg(0x100e20);
436	ram->hwsq.r_0x100e24 = hwsq_reg(0x100e24);
437	ram->hwsq.r_0x611200 = hwsq_reg(0x611200);
438
439	for (i = 0; i < 9; i++)
440		ram->hwsq.r_timing[i] = hwsq_reg(0x100220 + (i * 0x04));
441
442	if (ram->base.ranks > 1) {
443		ram->hwsq.r_mr[0] = hwsq_reg2(0x1002c0, 0x1002c8);
444		ram->hwsq.r_mr[1] = hwsq_reg2(0x1002c4, 0x1002cc);
445		ram->hwsq.r_mr[2] = hwsq_reg2(0x1002e0, 0x1002e8);
446		ram->hwsq.r_mr[3] = hwsq_reg2(0x1002e4, 0x1002ec);
447	} else {
448		ram->hwsq.r_mr[0] = hwsq_reg(0x1002c0);
449		ram->hwsq.r_mr[1] = hwsq_reg(0x1002c4);
450		ram->hwsq.r_mr[2] = hwsq_reg(0x1002e0);
451		ram->hwsq.r_mr[3] = hwsq_reg(0x1002e4);
452	}
453
454	return 0;
455}
456
457struct nvkm_oclass
458nv50_ram_oclass = {
459	.ofuncs = &(struct nvkm_ofuncs) {
460		.ctor = nv50_ram_ctor,
461		.dtor = _nvkm_ram_dtor,
462		.init = _nvkm_ram_init,
463		.fini = _nvkm_ram_fini,
464	}
465};
466