Lines Matching refs:priv
77 gf100_fifo_runlist_update(struct gf100_fifo_priv *priv) in gf100_fifo_runlist_update() argument
79 struct nvkm_bar *bar = nvkm_bar(priv); in gf100_fifo_runlist_update()
83 mutex_lock(&nv_subdev(priv)->mutex); in gf100_fifo_runlist_update()
84 cur = priv->runlist.mem[priv->runlist.active]; in gf100_fifo_runlist_update()
85 priv->runlist.active = !priv->runlist.active; in gf100_fifo_runlist_update()
88 struct gf100_fifo_chan *chan = (void *)priv->base.channel[i]; in gf100_fifo_runlist_update()
97 nv_wr32(priv, 0x002270, cur->addr >> 12); in gf100_fifo_runlist_update()
98 nv_wr32(priv, 0x002274, 0x01f00000 | (p >> 3)); in gf100_fifo_runlist_update()
100 if (wait_event_timeout(priv->runlist.wait, in gf100_fifo_runlist_update()
101 !(nv_rd32(priv, 0x00227c) & 0x00100000), in gf100_fifo_runlist_update()
103 nv_error(priv, "runlist update timeout\n"); in gf100_fifo_runlist_update()
104 mutex_unlock(&nv_subdev(priv)->mutex); in gf100_fifo_runlist_update()
149 struct gf100_fifo_priv *priv = (void *)parent->engine; in gf100_fifo_context_detach() local
166 nv_wr32(priv, 0x002634, chan->base.chid); in gf100_fifo_context_detach()
167 if (!nv_wait(priv, 0x002634, 0xffffffff, chan->base.chid)) { in gf100_fifo_context_detach()
168 nv_error(priv, "channel %d [%s] kick timeout\n", in gf100_fifo_context_detach()
189 struct gf100_fifo_priv *priv = (void *)engine; in gf100_fifo_chan_ctor() local
205 priv->user.bar.offset, 0x1000, in gf100_fifo_chan_ctor()
228 nv_wo32(priv->user.mem, usermem + i, 0x00000000); in gf100_fifo_chan_ctor()
230 nv_wo32(base, 0x08, lower_32_bits(priv->user.mem->addr + usermem)); in gf100_fifo_chan_ctor()
231 nv_wo32(base, 0x0c, upper_32_bits(priv->user.mem->addr + usermem)); in gf100_fifo_chan_ctor()
254 struct gf100_fifo_priv *priv = (void *)object->engine; in gf100_fifo_chan_init() local
263 nv_wr32(priv, 0x003000 + (chid * 8), 0xc0000000 | base->addr >> 12); in gf100_fifo_chan_init()
266 nv_wr32(priv, 0x003004 + (chid * 8), 0x001f0001); in gf100_fifo_chan_init()
267 gf100_fifo_runlist_update(priv); in gf100_fifo_chan_init()
273 static void gf100_fifo_intr_engine(struct gf100_fifo_priv *priv);
278 struct gf100_fifo_priv *priv = (void *)object->engine; in gf100_fifo_chan_fini() local
283 nv_mask(priv, 0x003004 + (chid * 8), 0x00000001, 0x00000000); in gf100_fifo_chan_fini()
284 gf100_fifo_runlist_update(priv); in gf100_fifo_chan_fini()
287 gf100_fifo_intr_engine(priv); in gf100_fifo_chan_fini()
289 nv_wr32(priv, 0x003000 + (chid * 8), 0x00000000); in gf100_fifo_chan_fini()
374 gf100_fifo_engidx(struct gf100_fifo_priv *priv, u32 engn) in gf100_fifo_engidx() argument
391 gf100_fifo_engine(struct gf100_fifo_priv *priv, u32 engn) in gf100_fifo_engine() argument
404 return nvkm_engine(priv, engn); in gf100_fifo_engine()
410 struct gf100_fifo_priv *priv = container_of(work, typeof(*priv), fault); in gf100_fifo_recover_work() local
416 spin_lock_irqsave(&priv->base.lock, flags); in gf100_fifo_recover_work()
417 mask = priv->mask; in gf100_fifo_recover_work()
418 priv->mask = 0ULL; in gf100_fifo_recover_work()
419 spin_unlock_irqrestore(&priv->base.lock, flags); in gf100_fifo_recover_work()
422 engm |= 1 << gf100_fifo_engidx(priv, engn); in gf100_fifo_recover_work()
423 nv_mask(priv, 0x002630, engm, engm); in gf100_fifo_recover_work()
426 if ((engine = (void *)nvkm_engine(priv, engn))) { in gf100_fifo_recover_work()
432 gf100_fifo_runlist_update(priv); in gf100_fifo_recover_work()
433 nv_wr32(priv, 0x00262c, engm); in gf100_fifo_recover_work()
434 nv_mask(priv, 0x002630, engm, 0x00000000); in gf100_fifo_recover_work()
438 gf100_fifo_recover(struct gf100_fifo_priv *priv, struct nvkm_engine *engine, in gf100_fifo_recover() argument
444 nv_error(priv, "%s engine fault on channel %d, recovering...\n", in gf100_fifo_recover()
447 nv_mask(priv, 0x003004 + (chid * 0x08), 0x00000001, 0x00000000); in gf100_fifo_recover()
450 spin_lock_irqsave(&priv->base.lock, flags); in gf100_fifo_recover()
451 priv->mask |= 1ULL << nv_engidx(engine); in gf100_fifo_recover()
452 spin_unlock_irqrestore(&priv->base.lock, flags); in gf100_fifo_recover()
453 schedule_work(&priv->fault); in gf100_fifo_recover()
457 gf100_fifo_swmthd(struct gf100_fifo_priv *priv, u32 chid, u32 mthd, u32 data) in gf100_fifo_swmthd() argument
464 spin_lock_irqsave(&priv->base.lock, flags); in gf100_fifo_swmthd()
465 if (likely(chid >= priv->base.min && chid <= priv->base.max)) in gf100_fifo_swmthd()
466 chan = (void *)priv->base.channel[chid]; in gf100_fifo_swmthd()
478 spin_unlock_irqrestore(&priv->base.lock, flags); in gf100_fifo_swmthd()
489 gf100_fifo_intr_sched_ctxsw(struct gf100_fifo_priv *priv) in gf100_fifo_intr_sched_ctxsw() argument
496 u32 stat = nv_rd32(priv, 0x002640 + (engn * 0x04)); in gf100_fifo_intr_sched_ctxsw()
505 if (!(chan = (void *)priv->base.channel[chid])) in gf100_fifo_intr_sched_ctxsw()
507 if (!(engine = gf100_fifo_engine(priv, engn))) in gf100_fifo_intr_sched_ctxsw()
509 gf100_fifo_recover(priv, engine, chan); in gf100_fifo_intr_sched_ctxsw()
515 gf100_fifo_intr_sched(struct gf100_fifo_priv *priv) in gf100_fifo_intr_sched() argument
517 u32 intr = nv_rd32(priv, 0x00254c); in gf100_fifo_intr_sched()
526 nv_error(priv, "SCHED_ERROR [ %s ]\n", en ? en->name : enunk); in gf100_fifo_intr_sched()
530 gf100_fifo_intr_sched_ctxsw(priv); in gf100_fifo_intr_sched()
597 gf100_fifo_intr_fault(struct gf100_fifo_priv *priv, int unit) in gf100_fifo_intr_fault() argument
599 u32 inst = nv_rd32(priv, 0x002800 + (unit * 0x10)); in gf100_fifo_intr_fault()
600 u32 valo = nv_rd32(priv, 0x002804 + (unit * 0x10)); in gf100_fifo_intr_fault()
601 u32 vahi = nv_rd32(priv, 0x002808 + (unit * 0x10)); in gf100_fifo_intr_fault()
602 u32 stat = nv_rd32(priv, 0x00280c + (unit * 0x10)); in gf100_fifo_intr_fault()
624 nv_mask(priv, 0x001704, 0x00000000, 0x00000000); in gf100_fifo_intr_fault()
627 nv_mask(priv, 0x001714, 0x00000000, 0x00000000); in gf100_fifo_intr_fault()
630 nv_mask(priv, 0x001718, 0x00000000, 0x00000000); in gf100_fifo_intr_fault()
633 engine = nvkm_engine(priv, eu->data2); in gf100_fifo_intr_fault()
652 nv_error(priv, "%s fault at 0x%010llx [%s] from %s/%s%s%s%s on " in gf100_fifo_intr_fault()
663 gf100_fifo_recover(priv, engine, (void *)object); in gf100_fifo_intr_fault()
681 gf100_fifo_intr_pbdma(struct gf100_fifo_priv *priv, int unit) in gf100_fifo_intr_pbdma() argument
683 u32 stat = nv_rd32(priv, 0x040108 + (unit * 0x2000)); in gf100_fifo_intr_pbdma()
684 u32 addr = nv_rd32(priv, 0x0400c0 + (unit * 0x2000)); in gf100_fifo_intr_pbdma()
685 u32 data = nv_rd32(priv, 0x0400c4 + (unit * 0x2000)); in gf100_fifo_intr_pbdma()
686 u32 chid = nv_rd32(priv, 0x040120 + (unit * 0x2000)) & 0x7f; in gf100_fifo_intr_pbdma()
692 if (!gf100_fifo_swmthd(priv, chid, mthd, data)) in gf100_fifo_intr_pbdma()
697 nv_error(priv, "PBDMA%d:", unit); in gf100_fifo_intr_pbdma()
700 nv_error(priv, in gf100_fifo_intr_pbdma()
703 nvkm_client_name_for_fifo_chid(&priv->base, chid), in gf100_fifo_intr_pbdma()
707 nv_wr32(priv, 0x0400c0 + (unit * 0x2000), 0x80600008); in gf100_fifo_intr_pbdma()
708 nv_wr32(priv, 0x040108 + (unit * 0x2000), stat); in gf100_fifo_intr_pbdma()
712 gf100_fifo_intr_runlist(struct gf100_fifo_priv *priv) in gf100_fifo_intr_runlist() argument
714 u32 intr = nv_rd32(priv, 0x002a00); in gf100_fifo_intr_runlist()
717 wake_up(&priv->runlist.wait); in gf100_fifo_intr_runlist()
718 nv_wr32(priv, 0x002a00, 0x10000000); in gf100_fifo_intr_runlist()
723 nv_error(priv, "RUNLIST 0x%08x\n", intr); in gf100_fifo_intr_runlist()
724 nv_wr32(priv, 0x002a00, intr); in gf100_fifo_intr_runlist()
729 gf100_fifo_intr_engine_unit(struct gf100_fifo_priv *priv, int engn) in gf100_fifo_intr_engine_unit() argument
731 u32 intr = nv_rd32(priv, 0x0025a8 + (engn * 0x04)); in gf100_fifo_intr_engine_unit()
732 u32 inte = nv_rd32(priv, 0x002628); in gf100_fifo_intr_engine_unit()
735 nv_wr32(priv, 0x0025a8 + (engn * 0x04), intr); in gf100_fifo_intr_engine_unit()
740 nvkm_fifo_uevent(&priv->base); in gf100_fifo_intr_engine_unit()
744 nv_error(priv, "ENGINE %d %d %01x", engn, unkn, ints); in gf100_fifo_intr_engine_unit()
745 nv_mask(priv, 0x002628, ints, 0); in gf100_fifo_intr_engine_unit()
751 gf100_fifo_intr_engine(struct gf100_fifo_priv *priv) in gf100_fifo_intr_engine() argument
753 u32 mask = nv_rd32(priv, 0x0025a4); in gf100_fifo_intr_engine()
756 gf100_fifo_intr_engine_unit(priv, unit); in gf100_fifo_intr_engine()
764 struct gf100_fifo_priv *priv = (void *)subdev; in gf100_fifo_intr() local
765 u32 mask = nv_rd32(priv, 0x002140); in gf100_fifo_intr()
766 u32 stat = nv_rd32(priv, 0x002100) & mask; in gf100_fifo_intr()
769 u32 intr = nv_rd32(priv, 0x00252c); in gf100_fifo_intr()
770 nv_warn(priv, "INTR 0x00000001: 0x%08x\n", intr); in gf100_fifo_intr()
771 nv_wr32(priv, 0x002100, 0x00000001); in gf100_fifo_intr()
776 gf100_fifo_intr_sched(priv); in gf100_fifo_intr()
777 nv_wr32(priv, 0x002100, 0x00000100); in gf100_fifo_intr()
782 u32 intr = nv_rd32(priv, 0x00256c); in gf100_fifo_intr()
783 nv_warn(priv, "INTR 0x00010000: 0x%08x\n", intr); in gf100_fifo_intr()
784 nv_wr32(priv, 0x002100, 0x00010000); in gf100_fifo_intr()
789 u32 intr = nv_rd32(priv, 0x00258c); in gf100_fifo_intr()
790 nv_warn(priv, "INTR 0x01000000: 0x%08x\n", intr); in gf100_fifo_intr()
791 nv_wr32(priv, 0x002100, 0x01000000); in gf100_fifo_intr()
796 u32 mask = nv_rd32(priv, 0x00259c); in gf100_fifo_intr()
799 gf100_fifo_intr_fault(priv, unit); in gf100_fifo_intr()
800 nv_wr32(priv, 0x00259c, (1 << unit)); in gf100_fifo_intr()
807 u32 mask = nv_rd32(priv, 0x0025a0); in gf100_fifo_intr()
810 gf100_fifo_intr_pbdma(priv, unit); in gf100_fifo_intr()
811 nv_wr32(priv, 0x0025a0, (1 << unit)); in gf100_fifo_intr()
818 gf100_fifo_intr_runlist(priv); in gf100_fifo_intr()
823 gf100_fifo_intr_engine(priv); in gf100_fifo_intr()
828 nv_error(priv, "INTR 0x%08x\n", stat); in gf100_fifo_intr()
829 nv_mask(priv, 0x002140, stat, 0x00000000); in gf100_fifo_intr()
830 nv_wr32(priv, 0x002100, stat); in gf100_fifo_intr()
860 struct gf100_fifo_priv *priv; in gf100_fifo_ctor() local
863 ret = nvkm_fifo_create(parent, engine, oclass, 0, 127, &priv); in gf100_fifo_ctor()
864 *pobject = nv_object(priv); in gf100_fifo_ctor()
868 INIT_WORK(&priv->fault, gf100_fifo_recover_work); in gf100_fifo_ctor()
870 ret = nvkm_gpuobj_new(nv_object(priv), NULL, 0x1000, 0x1000, 0, in gf100_fifo_ctor()
871 &priv->runlist.mem[0]); in gf100_fifo_ctor()
875 ret = nvkm_gpuobj_new(nv_object(priv), NULL, 0x1000, 0x1000, 0, in gf100_fifo_ctor()
876 &priv->runlist.mem[1]); in gf100_fifo_ctor()
880 init_waitqueue_head(&priv->runlist.wait); in gf100_fifo_ctor()
882 ret = nvkm_gpuobj_new(nv_object(priv), NULL, 128 * 0x1000, 0x1000, 0, in gf100_fifo_ctor()
883 &priv->user.mem); in gf100_fifo_ctor()
887 ret = nvkm_gpuobj_map(priv->user.mem, NV_MEM_ACCESS_RW, in gf100_fifo_ctor()
888 &priv->user.bar); in gf100_fifo_ctor()
892 ret = nvkm_event_init(&gf100_fifo_uevent_func, 1, 1, &priv->base.uevent); in gf100_fifo_ctor()
896 nv_subdev(priv)->unit = 0x00000100; in gf100_fifo_ctor()
897 nv_subdev(priv)->intr = gf100_fifo_intr; in gf100_fifo_ctor()
898 nv_engine(priv)->cclass = &gf100_fifo_cclass; in gf100_fifo_ctor()
899 nv_engine(priv)->sclass = gf100_fifo_sclass; in gf100_fifo_ctor()
906 struct gf100_fifo_priv *priv = (void *)object; in gf100_fifo_dtor() local
908 nvkm_gpuobj_unmap(&priv->user.bar); in gf100_fifo_dtor()
909 nvkm_gpuobj_ref(NULL, &priv->user.mem); in gf100_fifo_dtor()
910 nvkm_gpuobj_ref(NULL, &priv->runlist.mem[0]); in gf100_fifo_dtor()
911 nvkm_gpuobj_ref(NULL, &priv->runlist.mem[1]); in gf100_fifo_dtor()
913 nvkm_fifo_destroy(&priv->base); in gf100_fifo_dtor()
919 struct gf100_fifo_priv *priv = (void *)object; in gf100_fifo_init() local
922 ret = nvkm_fifo_init(&priv->base); in gf100_fifo_init()
926 nv_wr32(priv, 0x000204, 0xffffffff); in gf100_fifo_init()
927 nv_wr32(priv, 0x002204, 0xffffffff); in gf100_fifo_init()
929 priv->spoon_nr = hweight32(nv_rd32(priv, 0x002204)); in gf100_fifo_init()
930 nv_debug(priv, "%d PBDMA unit(s)\n", priv->spoon_nr); in gf100_fifo_init()
933 if (priv->spoon_nr >= 3) { in gf100_fifo_init()
934 nv_wr32(priv, 0x002208, ~(1 << 0)); /* PGRAPH */ in gf100_fifo_init()
935 nv_wr32(priv, 0x00220c, ~(1 << 1)); /* PVP */ in gf100_fifo_init()
936 nv_wr32(priv, 0x002210, ~(1 << 1)); /* PMSPP */ in gf100_fifo_init()
937 nv_wr32(priv, 0x002214, ~(1 << 1)); /* PMSVLD */ in gf100_fifo_init()
938 nv_wr32(priv, 0x002218, ~(1 << 2)); /* PCE0 */ in gf100_fifo_init()
939 nv_wr32(priv, 0x00221c, ~(1 << 1)); /* PCE1 */ in gf100_fifo_init()
943 for (i = 0; i < priv->spoon_nr; i++) { in gf100_fifo_init()
944 nv_mask(priv, 0x04013c + (i * 0x2000), 0x10000100, 0x00000000); in gf100_fifo_init()
945 nv_wr32(priv, 0x040108 + (i * 0x2000), 0xffffffff); /* INTR */ in gf100_fifo_init()
946 nv_wr32(priv, 0x04010c + (i * 0x2000), 0xfffffeff); /* INTREN */ in gf100_fifo_init()
949 nv_mask(priv, 0x002200, 0x00000001, 0x00000001); in gf100_fifo_init()
950 nv_wr32(priv, 0x002254, 0x10000000 | priv->user.bar.offset >> 12); in gf100_fifo_init()
952 nv_wr32(priv, 0x002100, 0xffffffff); in gf100_fifo_init()
953 nv_wr32(priv, 0x002140, 0x7fffffff); in gf100_fifo_init()
954 nv_wr32(priv, 0x002628, 0x00000001); /* ENGINE_INTR_EN */ in gf100_fifo_init()