1/* 2 * Universal Flash Storage Host controller driver Core 3 * 4 * This code is based on drivers/scsi/ufs/ufshcd.c 5 * Copyright (C) 2011-2013 Samsung India Software Operations 6 * Copyright (c) 2013-2014, The Linux Foundation. All rights reserved. 7 * 8 * Authors: 9 * Santosh Yaraganavi <santosh.sy@samsung.com> 10 * Vinayak Holikatti <h.vinayak@samsung.com> 11 * 12 * This program is free software; you can redistribute it and/or 13 * modify it under the terms of the GNU General Public License 14 * as published by the Free Software Foundation; either version 2 15 * of the License, or (at your option) any later version. 16 * See the COPYING file in the top-level directory or visit 17 * <http://www.gnu.org/licenses/gpl-2.0.html> 18 * 19 * This program is distributed in the hope that it will be useful, 20 * but WITHOUT ANY WARRANTY; without even the implied warranty of 21 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 22 * GNU General Public License for more details. 23 * 24 * This program is provided "AS IS" and "WITH ALL FAULTS" and 25 * without warranty of any kind. You are solely responsible for 26 * determining the appropriateness of using and distributing 27 * the program and assume all risks associated with your exercise 28 * of rights with respect to the program, including but not limited 29 * to infringement of third party rights, the risks and costs of 30 * program errors, damage to or loss of data, programs or equipment, 31 * and unavailability or interruption of operations. Under no 32 * circumstances will the contributor of this Program be liable for 33 * any damages of any kind arising from your use or distribution of 34 * this program. 35 * 36 * The Linux Foundation chooses to take subject only to the GPLv2 37 * license terms, and distributes only under these terms. 38 */ 39 40#include <linux/async.h> 41#include <linux/devfreq.h> 42 43#include "ufshcd.h" 44#include "unipro.h" 45 46#define UFSHCD_ENABLE_INTRS (UTP_TRANSFER_REQ_COMPL |\ 47 UTP_TASK_REQ_COMPL |\ 48 UFSHCD_ERROR_MASK) 49/* UIC command timeout, unit: ms */ 50#define UIC_CMD_TIMEOUT 500 51 52/* NOP OUT retries waiting for NOP IN response */ 53#define NOP_OUT_RETRIES 10 54/* Timeout after 30 msecs if NOP OUT hangs without response */ 55#define NOP_OUT_TIMEOUT 30 /* msecs */ 56 57/* Query request retries */ 58#define QUERY_REQ_RETRIES 10 59/* Query request timeout */ 60#define QUERY_REQ_TIMEOUT 30 /* msec */ 61 62/* Task management command timeout */ 63#define TM_CMD_TIMEOUT 100 /* msecs */ 64 65/* maximum number of link-startup retries */ 66#define DME_LINKSTARTUP_RETRIES 3 67 68/* maximum number of reset retries before giving up */ 69#define MAX_HOST_RESET_RETRIES 5 70 71/* Expose the flag value from utp_upiu_query.value */ 72#define MASK_QUERY_UPIU_FLAG_LOC 0xFF 73 74/* Interrupt aggregation default timeout, unit: 40us */ 75#define INT_AGGR_DEF_TO 0x02 76 77#define ufshcd_toggle_vreg(_dev, _vreg, _on) \ 78 ({ \ 79 int _ret; \ 80 if (_on) \ 81 _ret = ufshcd_enable_vreg(_dev, _vreg); \ 82 else \ 83 _ret = ufshcd_disable_vreg(_dev, _vreg); \ 84 _ret; \ 85 }) 86 87static u32 ufs_query_desc_max_size[] = { 88 QUERY_DESC_DEVICE_MAX_SIZE, 89 QUERY_DESC_CONFIGURAION_MAX_SIZE, 90 QUERY_DESC_UNIT_MAX_SIZE, 91 QUERY_DESC_RFU_MAX_SIZE, 92 QUERY_DESC_INTERCONNECT_MAX_SIZE, 93 QUERY_DESC_STRING_MAX_SIZE, 94 QUERY_DESC_RFU_MAX_SIZE, 95 QUERY_DESC_GEOMETRY_MAZ_SIZE, 96 QUERY_DESC_POWER_MAX_SIZE, 97 QUERY_DESC_RFU_MAX_SIZE, 98}; 99 100enum { 101 UFSHCD_MAX_CHANNEL = 0, 102 UFSHCD_MAX_ID = 1, 103 UFSHCD_CMD_PER_LUN = 32, 104 UFSHCD_CAN_QUEUE = 32, 105}; 106 107/* UFSHCD states */ 108enum { 109 UFSHCD_STATE_RESET, 110 UFSHCD_STATE_ERROR, 111 UFSHCD_STATE_OPERATIONAL, 112}; 113 114/* UFSHCD error handling flags */ 115enum { 116 UFSHCD_EH_IN_PROGRESS = (1 << 0), 117}; 118 119/* UFSHCD UIC layer error flags */ 120enum { 121 UFSHCD_UIC_DL_PA_INIT_ERROR = (1 << 0), /* Data link layer error */ 122 UFSHCD_UIC_NL_ERROR = (1 << 1), /* Network layer error */ 123 UFSHCD_UIC_TL_ERROR = (1 << 2), /* Transport Layer error */ 124 UFSHCD_UIC_DME_ERROR = (1 << 3), /* DME error */ 125}; 126 127/* Interrupt configuration options */ 128enum { 129 UFSHCD_INT_DISABLE, 130 UFSHCD_INT_ENABLE, 131 UFSHCD_INT_CLEAR, 132}; 133 134#define ufshcd_set_eh_in_progress(h) \ 135 (h->eh_flags |= UFSHCD_EH_IN_PROGRESS) 136#define ufshcd_eh_in_progress(h) \ 137 (h->eh_flags & UFSHCD_EH_IN_PROGRESS) 138#define ufshcd_clear_eh_in_progress(h) \ 139 (h->eh_flags &= ~UFSHCD_EH_IN_PROGRESS) 140 141#define ufshcd_set_ufs_dev_active(h) \ 142 ((h)->curr_dev_pwr_mode = UFS_ACTIVE_PWR_MODE) 143#define ufshcd_set_ufs_dev_sleep(h) \ 144 ((h)->curr_dev_pwr_mode = UFS_SLEEP_PWR_MODE) 145#define ufshcd_set_ufs_dev_poweroff(h) \ 146 ((h)->curr_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE) 147#define ufshcd_is_ufs_dev_active(h) \ 148 ((h)->curr_dev_pwr_mode == UFS_ACTIVE_PWR_MODE) 149#define ufshcd_is_ufs_dev_sleep(h) \ 150 ((h)->curr_dev_pwr_mode == UFS_SLEEP_PWR_MODE) 151#define ufshcd_is_ufs_dev_poweroff(h) \ 152 ((h)->curr_dev_pwr_mode == UFS_POWERDOWN_PWR_MODE) 153 154static struct ufs_pm_lvl_states ufs_pm_lvl_states[] = { 155 {UFS_ACTIVE_PWR_MODE, UIC_LINK_ACTIVE_STATE}, 156 {UFS_ACTIVE_PWR_MODE, UIC_LINK_HIBERN8_STATE}, 157 {UFS_SLEEP_PWR_MODE, UIC_LINK_ACTIVE_STATE}, 158 {UFS_SLEEP_PWR_MODE, UIC_LINK_HIBERN8_STATE}, 159 {UFS_POWERDOWN_PWR_MODE, UIC_LINK_HIBERN8_STATE}, 160 {UFS_POWERDOWN_PWR_MODE, UIC_LINK_OFF_STATE}, 161}; 162 163static inline enum ufs_dev_pwr_mode 164ufs_get_pm_lvl_to_dev_pwr_mode(enum ufs_pm_level lvl) 165{ 166 return ufs_pm_lvl_states[lvl].dev_state; 167} 168 169static inline enum uic_link_state 170ufs_get_pm_lvl_to_link_pwr_state(enum ufs_pm_level lvl) 171{ 172 return ufs_pm_lvl_states[lvl].link_state; 173} 174 175static void ufshcd_tmc_handler(struct ufs_hba *hba); 176static void ufshcd_async_scan(void *data, async_cookie_t cookie); 177static int ufshcd_reset_and_restore(struct ufs_hba *hba); 178static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag); 179static void ufshcd_hba_exit(struct ufs_hba *hba); 180static int ufshcd_probe_hba(struct ufs_hba *hba); 181static int __ufshcd_setup_clocks(struct ufs_hba *hba, bool on, 182 bool skip_ref_clk); 183static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on); 184static int ufshcd_uic_hibern8_exit(struct ufs_hba *hba); 185static int ufshcd_uic_hibern8_enter(struct ufs_hba *hba); 186static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba); 187static int ufshcd_host_reset_and_restore(struct ufs_hba *hba); 188static irqreturn_t ufshcd_intr(int irq, void *__hba); 189static int ufshcd_config_pwr_mode(struct ufs_hba *hba, 190 struct ufs_pa_layer_attr *desired_pwr_mode); 191 192static inline int ufshcd_enable_irq(struct ufs_hba *hba) 193{ 194 int ret = 0; 195 196 if (!hba->is_irq_enabled) { 197 ret = request_irq(hba->irq, ufshcd_intr, IRQF_SHARED, UFSHCD, 198 hba); 199 if (ret) 200 dev_err(hba->dev, "%s: request_irq failed, ret=%d\n", 201 __func__, ret); 202 hba->is_irq_enabled = true; 203 } 204 205 return ret; 206} 207 208static inline void ufshcd_disable_irq(struct ufs_hba *hba) 209{ 210 if (hba->is_irq_enabled) { 211 free_irq(hba->irq, hba); 212 hba->is_irq_enabled = false; 213 } 214} 215 216/* 217 * ufshcd_wait_for_register - wait for register value to change 218 * @hba - per-adapter interface 219 * @reg - mmio register offset 220 * @mask - mask to apply to read register value 221 * @val - wait condition 222 * @interval_us - polling interval in microsecs 223 * @timeout_ms - timeout in millisecs 224 * 225 * Returns -ETIMEDOUT on error, zero on success 226 */ 227static int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask, 228 u32 val, unsigned long interval_us, unsigned long timeout_ms) 229{ 230 int err = 0; 231 unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms); 232 233 /* ignore bits that we don't intend to wait on */ 234 val = val & mask; 235 236 while ((ufshcd_readl(hba, reg) & mask) != val) { 237 /* wakeup within 50us of expiry */ 238 usleep_range(interval_us, interval_us + 50); 239 240 if (time_after(jiffies, timeout)) { 241 if ((ufshcd_readl(hba, reg) & mask) != val) 242 err = -ETIMEDOUT; 243 break; 244 } 245 } 246 247 return err; 248} 249 250/** 251 * ufshcd_get_intr_mask - Get the interrupt bit mask 252 * @hba - Pointer to adapter instance 253 * 254 * Returns interrupt bit mask per version 255 */ 256static inline u32 ufshcd_get_intr_mask(struct ufs_hba *hba) 257{ 258 if (hba->ufs_version == UFSHCI_VERSION_10) 259 return INTERRUPT_MASK_ALL_VER_10; 260 else 261 return INTERRUPT_MASK_ALL_VER_11; 262} 263 264/** 265 * ufshcd_get_ufs_version - Get the UFS version supported by the HBA 266 * @hba - Pointer to adapter instance 267 * 268 * Returns UFSHCI version supported by the controller 269 */ 270static inline u32 ufshcd_get_ufs_version(struct ufs_hba *hba) 271{ 272 return ufshcd_readl(hba, REG_UFS_VERSION); 273} 274 275/** 276 * ufshcd_is_device_present - Check if any device connected to 277 * the host controller 278 * @hba: pointer to adapter instance 279 * 280 * Returns 1 if device present, 0 if no device detected 281 */ 282static inline int ufshcd_is_device_present(struct ufs_hba *hba) 283{ 284 return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) & 285 DEVICE_PRESENT) ? 1 : 0; 286} 287 288/** 289 * ufshcd_get_tr_ocs - Get the UTRD Overall Command Status 290 * @lrb: pointer to local command reference block 291 * 292 * This function is used to get the OCS field from UTRD 293 * Returns the OCS field in the UTRD 294 */ 295static inline int ufshcd_get_tr_ocs(struct ufshcd_lrb *lrbp) 296{ 297 return le32_to_cpu(lrbp->utr_descriptor_ptr->header.dword_2) & MASK_OCS; 298} 299 300/** 301 * ufshcd_get_tmr_ocs - Get the UTMRD Overall Command Status 302 * @task_req_descp: pointer to utp_task_req_desc structure 303 * 304 * This function is used to get the OCS field from UTMRD 305 * Returns the OCS field in the UTMRD 306 */ 307static inline int 308ufshcd_get_tmr_ocs(struct utp_task_req_desc *task_req_descp) 309{ 310 return le32_to_cpu(task_req_descp->header.dword_2) & MASK_OCS; 311} 312 313/** 314 * ufshcd_get_tm_free_slot - get a free slot for task management request 315 * @hba: per adapter instance 316 * @free_slot: pointer to variable with available slot value 317 * 318 * Get a free tag and lock it until ufshcd_put_tm_slot() is called. 319 * Returns 0 if free slot is not available, else return 1 with tag value 320 * in @free_slot. 321 */ 322static bool ufshcd_get_tm_free_slot(struct ufs_hba *hba, int *free_slot) 323{ 324 int tag; 325 bool ret = false; 326 327 if (!free_slot) 328 goto out; 329 330 do { 331 tag = find_first_zero_bit(&hba->tm_slots_in_use, hba->nutmrs); 332 if (tag >= hba->nutmrs) 333 goto out; 334 } while (test_and_set_bit_lock(tag, &hba->tm_slots_in_use)); 335 336 *free_slot = tag; 337 ret = true; 338out: 339 return ret; 340} 341 342static inline void ufshcd_put_tm_slot(struct ufs_hba *hba, int slot) 343{ 344 clear_bit_unlock(slot, &hba->tm_slots_in_use); 345} 346 347/** 348 * ufshcd_utrl_clear - Clear a bit in UTRLCLR register 349 * @hba: per adapter instance 350 * @pos: position of the bit to be cleared 351 */ 352static inline void ufshcd_utrl_clear(struct ufs_hba *hba, u32 pos) 353{ 354 ufshcd_writel(hba, ~(1 << pos), REG_UTP_TRANSFER_REQ_LIST_CLEAR); 355} 356 357/** 358 * ufshcd_get_lists_status - Check UCRDY, UTRLRDY and UTMRLRDY 359 * @reg: Register value of host controller status 360 * 361 * Returns integer, 0 on Success and positive value if failed 362 */ 363static inline int ufshcd_get_lists_status(u32 reg) 364{ 365 /* 366 * The mask 0xFF is for the following HCS register bits 367 * Bit Description 368 * 0 Device Present 369 * 1 UTRLRDY 370 * 2 UTMRLRDY 371 * 3 UCRDY 372 * 4 HEI 373 * 5 DEI 374 * 6-7 reserved 375 */ 376 return (((reg) & (0xFF)) >> 1) ^ (0x07); 377} 378 379/** 380 * ufshcd_get_uic_cmd_result - Get the UIC command result 381 * @hba: Pointer to adapter instance 382 * 383 * This function gets the result of UIC command completion 384 * Returns 0 on success, non zero value on error 385 */ 386static inline int ufshcd_get_uic_cmd_result(struct ufs_hba *hba) 387{ 388 return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2) & 389 MASK_UIC_COMMAND_RESULT; 390} 391 392/** 393 * ufshcd_get_dme_attr_val - Get the value of attribute returned by UIC command 394 * @hba: Pointer to adapter instance 395 * 396 * This function gets UIC command argument3 397 * Returns 0 on success, non zero value on error 398 */ 399static inline u32 ufshcd_get_dme_attr_val(struct ufs_hba *hba) 400{ 401 return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3); 402} 403 404/** 405 * ufshcd_get_req_rsp - returns the TR response transaction type 406 * @ucd_rsp_ptr: pointer to response UPIU 407 */ 408static inline int 409ufshcd_get_req_rsp(struct utp_upiu_rsp *ucd_rsp_ptr) 410{ 411 return be32_to_cpu(ucd_rsp_ptr->header.dword_0) >> 24; 412} 413 414/** 415 * ufshcd_get_rsp_upiu_result - Get the result from response UPIU 416 * @ucd_rsp_ptr: pointer to response UPIU 417 * 418 * This function gets the response status and scsi_status from response UPIU 419 * Returns the response result code. 420 */ 421static inline int 422ufshcd_get_rsp_upiu_result(struct utp_upiu_rsp *ucd_rsp_ptr) 423{ 424 return be32_to_cpu(ucd_rsp_ptr->header.dword_1) & MASK_RSP_UPIU_RESULT; 425} 426 427/* 428 * ufshcd_get_rsp_upiu_data_seg_len - Get the data segment length 429 * from response UPIU 430 * @ucd_rsp_ptr: pointer to response UPIU 431 * 432 * Return the data segment length. 433 */ 434static inline unsigned int 435ufshcd_get_rsp_upiu_data_seg_len(struct utp_upiu_rsp *ucd_rsp_ptr) 436{ 437 return be32_to_cpu(ucd_rsp_ptr->header.dword_2) & 438 MASK_RSP_UPIU_DATA_SEG_LEN; 439} 440 441/** 442 * ufshcd_is_exception_event - Check if the device raised an exception event 443 * @ucd_rsp_ptr: pointer to response UPIU 444 * 445 * The function checks if the device raised an exception event indicated in 446 * the Device Information field of response UPIU. 447 * 448 * Returns true if exception is raised, false otherwise. 449 */ 450static inline bool ufshcd_is_exception_event(struct utp_upiu_rsp *ucd_rsp_ptr) 451{ 452 return be32_to_cpu(ucd_rsp_ptr->header.dword_2) & 453 MASK_RSP_EXCEPTION_EVENT ? true : false; 454} 455 456/** 457 * ufshcd_reset_intr_aggr - Reset interrupt aggregation values. 458 * @hba: per adapter instance 459 */ 460static inline void 461ufshcd_reset_intr_aggr(struct ufs_hba *hba) 462{ 463 ufshcd_writel(hba, INT_AGGR_ENABLE | 464 INT_AGGR_COUNTER_AND_TIMER_RESET, 465 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL); 466} 467 468/** 469 * ufshcd_config_intr_aggr - Configure interrupt aggregation values. 470 * @hba: per adapter instance 471 * @cnt: Interrupt aggregation counter threshold 472 * @tmout: Interrupt aggregation timeout value 473 */ 474static inline void 475ufshcd_config_intr_aggr(struct ufs_hba *hba, u8 cnt, u8 tmout) 476{ 477 ufshcd_writel(hba, INT_AGGR_ENABLE | INT_AGGR_PARAM_WRITE | 478 INT_AGGR_COUNTER_THLD_VAL(cnt) | 479 INT_AGGR_TIMEOUT_VAL(tmout), 480 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL); 481} 482 483/** 484 * ufshcd_enable_run_stop_reg - Enable run-stop registers, 485 * When run-stop registers are set to 1, it indicates the 486 * host controller that it can process the requests 487 * @hba: per adapter instance 488 */ 489static void ufshcd_enable_run_stop_reg(struct ufs_hba *hba) 490{ 491 ufshcd_writel(hba, UTP_TASK_REQ_LIST_RUN_STOP_BIT, 492 REG_UTP_TASK_REQ_LIST_RUN_STOP); 493 ufshcd_writel(hba, UTP_TRANSFER_REQ_LIST_RUN_STOP_BIT, 494 REG_UTP_TRANSFER_REQ_LIST_RUN_STOP); 495} 496 497/** 498 * ufshcd_hba_start - Start controller initialization sequence 499 * @hba: per adapter instance 500 */ 501static inline void ufshcd_hba_start(struct ufs_hba *hba) 502{ 503 ufshcd_writel(hba, CONTROLLER_ENABLE, REG_CONTROLLER_ENABLE); 504} 505 506/** 507 * ufshcd_is_hba_active - Get controller state 508 * @hba: per adapter instance 509 * 510 * Returns zero if controller is active, 1 otherwise 511 */ 512static inline int ufshcd_is_hba_active(struct ufs_hba *hba) 513{ 514 return (ufshcd_readl(hba, REG_CONTROLLER_ENABLE) & 0x1) ? 0 : 1; 515} 516 517static void ufshcd_ungate_work(struct work_struct *work) 518{ 519 int ret; 520 unsigned long flags; 521 struct ufs_hba *hba = container_of(work, struct ufs_hba, 522 clk_gating.ungate_work); 523 524 cancel_delayed_work_sync(&hba->clk_gating.gate_work); 525 526 spin_lock_irqsave(hba->host->host_lock, flags); 527 if (hba->clk_gating.state == CLKS_ON) { 528 spin_unlock_irqrestore(hba->host->host_lock, flags); 529 goto unblock_reqs; 530 } 531 532 spin_unlock_irqrestore(hba->host->host_lock, flags); 533 ufshcd_setup_clocks(hba, true); 534 535 /* Exit from hibern8 */ 536 if (ufshcd_can_hibern8_during_gating(hba)) { 537 /* Prevent gating in this path */ 538 hba->clk_gating.is_suspended = true; 539 if (ufshcd_is_link_hibern8(hba)) { 540 ret = ufshcd_uic_hibern8_exit(hba); 541 if (ret) 542 dev_err(hba->dev, "%s: hibern8 exit failed %d\n", 543 __func__, ret); 544 else 545 ufshcd_set_link_active(hba); 546 } 547 hba->clk_gating.is_suspended = false; 548 } 549unblock_reqs: 550 if (ufshcd_is_clkscaling_enabled(hba)) 551 devfreq_resume_device(hba->devfreq); 552 scsi_unblock_requests(hba->host); 553} 554 555/** 556 * ufshcd_hold - Enable clocks that were gated earlier due to ufshcd_release. 557 * Also, exit from hibern8 mode and set the link as active. 558 * @hba: per adapter instance 559 * @async: This indicates whether caller should ungate clocks asynchronously. 560 */ 561int ufshcd_hold(struct ufs_hba *hba, bool async) 562{ 563 int rc = 0; 564 unsigned long flags; 565 566 if (!ufshcd_is_clkgating_allowed(hba)) 567 goto out; 568 spin_lock_irqsave(hba->host->host_lock, flags); 569 hba->clk_gating.active_reqs++; 570 571start: 572 switch (hba->clk_gating.state) { 573 case CLKS_ON: 574 break; 575 case REQ_CLKS_OFF: 576 if (cancel_delayed_work(&hba->clk_gating.gate_work)) { 577 hba->clk_gating.state = CLKS_ON; 578 break; 579 } 580 /* 581 * If we here, it means gating work is either done or 582 * currently running. Hence, fall through to cancel gating 583 * work and to enable clocks. 584 */ 585 case CLKS_OFF: 586 scsi_block_requests(hba->host); 587 hba->clk_gating.state = REQ_CLKS_ON; 588 schedule_work(&hba->clk_gating.ungate_work); 589 /* 590 * fall through to check if we should wait for this 591 * work to be done or not. 592 */ 593 case REQ_CLKS_ON: 594 if (async) { 595 rc = -EAGAIN; 596 hba->clk_gating.active_reqs--; 597 break; 598 } 599 600 spin_unlock_irqrestore(hba->host->host_lock, flags); 601 flush_work(&hba->clk_gating.ungate_work); 602 /* Make sure state is CLKS_ON before returning */ 603 spin_lock_irqsave(hba->host->host_lock, flags); 604 goto start; 605 default: 606 dev_err(hba->dev, "%s: clk gating is in invalid state %d\n", 607 __func__, hba->clk_gating.state); 608 break; 609 } 610 spin_unlock_irqrestore(hba->host->host_lock, flags); 611out: 612 return rc; 613} 614 615static void ufshcd_gate_work(struct work_struct *work) 616{ 617 struct ufs_hba *hba = container_of(work, struct ufs_hba, 618 clk_gating.gate_work.work); 619 unsigned long flags; 620 621 spin_lock_irqsave(hba->host->host_lock, flags); 622 if (hba->clk_gating.is_suspended) { 623 hba->clk_gating.state = CLKS_ON; 624 goto rel_lock; 625 } 626 627 if (hba->clk_gating.active_reqs 628 || hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL 629 || hba->lrb_in_use || hba->outstanding_tasks 630 || hba->active_uic_cmd || hba->uic_async_done) 631 goto rel_lock; 632 633 spin_unlock_irqrestore(hba->host->host_lock, flags); 634 635 /* put the link into hibern8 mode before turning off clocks */ 636 if (ufshcd_can_hibern8_during_gating(hba)) { 637 if (ufshcd_uic_hibern8_enter(hba)) { 638 hba->clk_gating.state = CLKS_ON; 639 goto out; 640 } 641 ufshcd_set_link_hibern8(hba); 642 } 643 644 if (ufshcd_is_clkscaling_enabled(hba)) { 645 devfreq_suspend_device(hba->devfreq); 646 hba->clk_scaling.window_start_t = 0; 647 } 648 649 if (!ufshcd_is_link_active(hba)) 650 ufshcd_setup_clocks(hba, false); 651 else 652 /* If link is active, device ref_clk can't be switched off */ 653 __ufshcd_setup_clocks(hba, false, true); 654 655 /* 656 * In case you are here to cancel this work the gating state 657 * would be marked as REQ_CLKS_ON. In this case keep the state 658 * as REQ_CLKS_ON which would anyway imply that clocks are off 659 * and a request to turn them on is pending. By doing this way, 660 * we keep the state machine in tact and this would ultimately 661 * prevent from doing cancel work multiple times when there are 662 * new requests arriving before the current cancel work is done. 663 */ 664 spin_lock_irqsave(hba->host->host_lock, flags); 665 if (hba->clk_gating.state == REQ_CLKS_OFF) 666 hba->clk_gating.state = CLKS_OFF; 667 668rel_lock: 669 spin_unlock_irqrestore(hba->host->host_lock, flags); 670out: 671 return; 672} 673 674/* host lock must be held before calling this variant */ 675static void __ufshcd_release(struct ufs_hba *hba) 676{ 677 if (!ufshcd_is_clkgating_allowed(hba)) 678 return; 679 680 hba->clk_gating.active_reqs--; 681 682 if (hba->clk_gating.active_reqs || hba->clk_gating.is_suspended 683 || hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL 684 || hba->lrb_in_use || hba->outstanding_tasks 685 || hba->active_uic_cmd || hba->uic_async_done) 686 return; 687 688 hba->clk_gating.state = REQ_CLKS_OFF; 689 schedule_delayed_work(&hba->clk_gating.gate_work, 690 msecs_to_jiffies(hba->clk_gating.delay_ms)); 691} 692 693void ufshcd_release(struct ufs_hba *hba) 694{ 695 unsigned long flags; 696 697 spin_lock_irqsave(hba->host->host_lock, flags); 698 __ufshcd_release(hba); 699 spin_unlock_irqrestore(hba->host->host_lock, flags); 700} 701 702static ssize_t ufshcd_clkgate_delay_show(struct device *dev, 703 struct device_attribute *attr, char *buf) 704{ 705 struct ufs_hba *hba = dev_get_drvdata(dev); 706 707 return snprintf(buf, PAGE_SIZE, "%lu\n", hba->clk_gating.delay_ms); 708} 709 710static ssize_t ufshcd_clkgate_delay_store(struct device *dev, 711 struct device_attribute *attr, const char *buf, size_t count) 712{ 713 struct ufs_hba *hba = dev_get_drvdata(dev); 714 unsigned long flags, value; 715 716 if (kstrtoul(buf, 0, &value)) 717 return -EINVAL; 718 719 spin_lock_irqsave(hba->host->host_lock, flags); 720 hba->clk_gating.delay_ms = value; 721 spin_unlock_irqrestore(hba->host->host_lock, flags); 722 return count; 723} 724 725static void ufshcd_init_clk_gating(struct ufs_hba *hba) 726{ 727 if (!ufshcd_is_clkgating_allowed(hba)) 728 return; 729 730 hba->clk_gating.delay_ms = 150; 731 INIT_DELAYED_WORK(&hba->clk_gating.gate_work, ufshcd_gate_work); 732 INIT_WORK(&hba->clk_gating.ungate_work, ufshcd_ungate_work); 733 734 hba->clk_gating.delay_attr.show = ufshcd_clkgate_delay_show; 735 hba->clk_gating.delay_attr.store = ufshcd_clkgate_delay_store; 736 sysfs_attr_init(&hba->clk_gating.delay_attr.attr); 737 hba->clk_gating.delay_attr.attr.name = "clkgate_delay_ms"; 738 hba->clk_gating.delay_attr.attr.mode = S_IRUGO | S_IWUSR; 739 if (device_create_file(hba->dev, &hba->clk_gating.delay_attr)) 740 dev_err(hba->dev, "Failed to create sysfs for clkgate_delay\n"); 741} 742 743static void ufshcd_exit_clk_gating(struct ufs_hba *hba) 744{ 745 if (!ufshcd_is_clkgating_allowed(hba)) 746 return; 747 device_remove_file(hba->dev, &hba->clk_gating.delay_attr); 748 cancel_work_sync(&hba->clk_gating.ungate_work); 749 cancel_delayed_work_sync(&hba->clk_gating.gate_work); 750} 751 752/* Must be called with host lock acquired */ 753static void ufshcd_clk_scaling_start_busy(struct ufs_hba *hba) 754{ 755 if (!ufshcd_is_clkscaling_enabled(hba)) 756 return; 757 758 if (!hba->clk_scaling.is_busy_started) { 759 hba->clk_scaling.busy_start_t = ktime_get(); 760 hba->clk_scaling.is_busy_started = true; 761 } 762} 763 764static void ufshcd_clk_scaling_update_busy(struct ufs_hba *hba) 765{ 766 struct ufs_clk_scaling *scaling = &hba->clk_scaling; 767 768 if (!ufshcd_is_clkscaling_enabled(hba)) 769 return; 770 771 if (!hba->outstanding_reqs && scaling->is_busy_started) { 772 scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(), 773 scaling->busy_start_t)); 774 scaling->busy_start_t = ktime_set(0, 0); 775 scaling->is_busy_started = false; 776 } 777} 778/** 779 * ufshcd_send_command - Send SCSI or device management commands 780 * @hba: per adapter instance 781 * @task_tag: Task tag of the command 782 */ 783static inline 784void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag) 785{ 786 ufshcd_clk_scaling_start_busy(hba); 787 __set_bit(task_tag, &hba->outstanding_reqs); 788 ufshcd_writel(hba, 1 << task_tag, REG_UTP_TRANSFER_REQ_DOOR_BELL); 789} 790 791/** 792 * ufshcd_copy_sense_data - Copy sense data in case of check condition 793 * @lrb - pointer to local reference block 794 */ 795static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp) 796{ 797 int len; 798 if (lrbp->sense_buffer && 799 ufshcd_get_rsp_upiu_data_seg_len(lrbp->ucd_rsp_ptr)) { 800 len = be16_to_cpu(lrbp->ucd_rsp_ptr->sr.sense_data_len); 801 memcpy(lrbp->sense_buffer, 802 lrbp->ucd_rsp_ptr->sr.sense_data, 803 min_t(int, len, SCSI_SENSE_BUFFERSIZE)); 804 } 805} 806 807/** 808 * ufshcd_copy_query_response() - Copy the Query Response and the data 809 * descriptor 810 * @hba: per adapter instance 811 * @lrb - pointer to local reference block 812 */ 813static 814int ufshcd_copy_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) 815{ 816 struct ufs_query_res *query_res = &hba->dev_cmd.query.response; 817 818 memcpy(&query_res->upiu_res, &lrbp->ucd_rsp_ptr->qr, QUERY_OSF_SIZE); 819 820 /* Get the descriptor */ 821 if (lrbp->ucd_rsp_ptr->qr.opcode == UPIU_QUERY_OPCODE_READ_DESC) { 822 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr + 823 GENERAL_UPIU_REQUEST_SIZE; 824 u16 resp_len; 825 u16 buf_len; 826 827 /* data segment length */ 828 resp_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) & 829 MASK_QUERY_DATA_SEG_LEN; 830 buf_len = be16_to_cpu( 831 hba->dev_cmd.query.request.upiu_req.length); 832 if (likely(buf_len >= resp_len)) { 833 memcpy(hba->dev_cmd.query.descriptor, descp, resp_len); 834 } else { 835 dev_warn(hba->dev, 836 "%s: Response size is bigger than buffer", 837 __func__); 838 return -EINVAL; 839 } 840 } 841 842 return 0; 843} 844 845/** 846 * ufshcd_hba_capabilities - Read controller capabilities 847 * @hba: per adapter instance 848 */ 849static inline void ufshcd_hba_capabilities(struct ufs_hba *hba) 850{ 851 hba->capabilities = ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES); 852 853 /* nutrs and nutmrs are 0 based values */ 854 hba->nutrs = (hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS) + 1; 855 hba->nutmrs = 856 ((hba->capabilities & MASK_TASK_MANAGEMENT_REQUEST_SLOTS) >> 16) + 1; 857} 858 859/** 860 * ufshcd_ready_for_uic_cmd - Check if controller is ready 861 * to accept UIC commands 862 * @hba: per adapter instance 863 * Return true on success, else false 864 */ 865static inline bool ufshcd_ready_for_uic_cmd(struct ufs_hba *hba) 866{ 867 if (ufshcd_readl(hba, REG_CONTROLLER_STATUS) & UIC_COMMAND_READY) 868 return true; 869 else 870 return false; 871} 872 873/** 874 * ufshcd_get_upmcrs - Get the power mode change request status 875 * @hba: Pointer to adapter instance 876 * 877 * This function gets the UPMCRS field of HCS register 878 * Returns value of UPMCRS field 879 */ 880static inline u8 ufshcd_get_upmcrs(struct ufs_hba *hba) 881{ 882 return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) >> 8) & 0x7; 883} 884 885/** 886 * ufshcd_dispatch_uic_cmd - Dispatch UIC commands to unipro layers 887 * @hba: per adapter instance 888 * @uic_cmd: UIC command 889 * 890 * Mutex must be held. 891 */ 892static inline void 893ufshcd_dispatch_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) 894{ 895 WARN_ON(hba->active_uic_cmd); 896 897 hba->active_uic_cmd = uic_cmd; 898 899 /* Write Args */ 900 ufshcd_writel(hba, uic_cmd->argument1, REG_UIC_COMMAND_ARG_1); 901 ufshcd_writel(hba, uic_cmd->argument2, REG_UIC_COMMAND_ARG_2); 902 ufshcd_writel(hba, uic_cmd->argument3, REG_UIC_COMMAND_ARG_3); 903 904 /* Write UIC Cmd */ 905 ufshcd_writel(hba, uic_cmd->command & COMMAND_OPCODE_MASK, 906 REG_UIC_COMMAND); 907} 908 909/** 910 * ufshcd_wait_for_uic_cmd - Wait complectioin of UIC command 911 * @hba: per adapter instance 912 * @uic_command: UIC command 913 * 914 * Must be called with mutex held. 915 * Returns 0 only if success. 916 */ 917static int 918ufshcd_wait_for_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) 919{ 920 int ret; 921 unsigned long flags; 922 923 if (wait_for_completion_timeout(&uic_cmd->done, 924 msecs_to_jiffies(UIC_CMD_TIMEOUT))) 925 ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT; 926 else 927 ret = -ETIMEDOUT; 928 929 spin_lock_irqsave(hba->host->host_lock, flags); 930 hba->active_uic_cmd = NULL; 931 spin_unlock_irqrestore(hba->host->host_lock, flags); 932 933 return ret; 934} 935 936/** 937 * __ufshcd_send_uic_cmd - Send UIC commands and retrieve the result 938 * @hba: per adapter instance 939 * @uic_cmd: UIC command 940 * 941 * Identical to ufshcd_send_uic_cmd() expect mutex. Must be called 942 * with mutex held and host_lock locked. 943 * Returns 0 only if success. 944 */ 945static int 946__ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) 947{ 948 if (!ufshcd_ready_for_uic_cmd(hba)) { 949 dev_err(hba->dev, 950 "Controller not ready to accept UIC commands\n"); 951 return -EIO; 952 } 953 954 init_completion(&uic_cmd->done); 955 956 ufshcd_dispatch_uic_cmd(hba, uic_cmd); 957 958 return 0; 959} 960 961/** 962 * ufshcd_send_uic_cmd - Send UIC commands and retrieve the result 963 * @hba: per adapter instance 964 * @uic_cmd: UIC command 965 * 966 * Returns 0 only if success. 967 */ 968static int 969ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) 970{ 971 int ret; 972 unsigned long flags; 973 974 ufshcd_hold(hba, false); 975 mutex_lock(&hba->uic_cmd_mutex); 976 ufshcd_add_delay_before_dme_cmd(hba); 977 978 spin_lock_irqsave(hba->host->host_lock, flags); 979 ret = __ufshcd_send_uic_cmd(hba, uic_cmd); 980 spin_unlock_irqrestore(hba->host->host_lock, flags); 981 if (!ret) 982 ret = ufshcd_wait_for_uic_cmd(hba, uic_cmd); 983 984 mutex_unlock(&hba->uic_cmd_mutex); 985 986 ufshcd_release(hba); 987 return ret; 988} 989 990/** 991 * ufshcd_map_sg - Map scatter-gather list to prdt 992 * @lrbp - pointer to local reference block 993 * 994 * Returns 0 in case of success, non-zero value in case of failure 995 */ 996static int ufshcd_map_sg(struct ufshcd_lrb *lrbp) 997{ 998 struct ufshcd_sg_entry *prd_table; 999 struct scatterlist *sg; 1000 struct scsi_cmnd *cmd; 1001 int sg_segments; 1002 int i; 1003 1004 cmd = lrbp->cmd; 1005 sg_segments = scsi_dma_map(cmd); 1006 if (sg_segments < 0) 1007 return sg_segments; 1008 1009 if (sg_segments) { 1010 lrbp->utr_descriptor_ptr->prd_table_length = 1011 cpu_to_le16((u16) (sg_segments)); 1012 1013 prd_table = (struct ufshcd_sg_entry *)lrbp->ucd_prdt_ptr; 1014 1015 scsi_for_each_sg(cmd, sg, sg_segments, i) { 1016 prd_table[i].size = 1017 cpu_to_le32(((u32) sg_dma_len(sg))-1); 1018 prd_table[i].base_addr = 1019 cpu_to_le32(lower_32_bits(sg->dma_address)); 1020 prd_table[i].upper_addr = 1021 cpu_to_le32(upper_32_bits(sg->dma_address)); 1022 } 1023 } else { 1024 lrbp->utr_descriptor_ptr->prd_table_length = 0; 1025 } 1026 1027 return 0; 1028} 1029 1030/** 1031 * ufshcd_enable_intr - enable interrupts 1032 * @hba: per adapter instance 1033 * @intrs: interrupt bits 1034 */ 1035static void ufshcd_enable_intr(struct ufs_hba *hba, u32 intrs) 1036{ 1037 u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE); 1038 1039 if (hba->ufs_version == UFSHCI_VERSION_10) { 1040 u32 rw; 1041 rw = set & INTERRUPT_MASK_RW_VER_10; 1042 set = rw | ((set ^ intrs) & intrs); 1043 } else { 1044 set |= intrs; 1045 } 1046 1047 ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE); 1048} 1049 1050/** 1051 * ufshcd_disable_intr - disable interrupts 1052 * @hba: per adapter instance 1053 * @intrs: interrupt bits 1054 */ 1055static void ufshcd_disable_intr(struct ufs_hba *hba, u32 intrs) 1056{ 1057 u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE); 1058 1059 if (hba->ufs_version == UFSHCI_VERSION_10) { 1060 u32 rw; 1061 rw = (set & INTERRUPT_MASK_RW_VER_10) & 1062 ~(intrs & INTERRUPT_MASK_RW_VER_10); 1063 set = rw | ((set & intrs) & ~INTERRUPT_MASK_RW_VER_10); 1064 1065 } else { 1066 set &= ~intrs; 1067 } 1068 1069 ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE); 1070} 1071 1072/** 1073 * ufshcd_prepare_req_desc_hdr() - Fills the requests header 1074 * descriptor according to request 1075 * @lrbp: pointer to local reference block 1076 * @upiu_flags: flags required in the header 1077 * @cmd_dir: requests data direction 1078 */ 1079static void ufshcd_prepare_req_desc_hdr(struct ufshcd_lrb *lrbp, 1080 u32 *upiu_flags, enum dma_data_direction cmd_dir) 1081{ 1082 struct utp_transfer_req_desc *req_desc = lrbp->utr_descriptor_ptr; 1083 u32 data_direction; 1084 u32 dword_0; 1085 1086 if (cmd_dir == DMA_FROM_DEVICE) { 1087 data_direction = UTP_DEVICE_TO_HOST; 1088 *upiu_flags = UPIU_CMD_FLAGS_READ; 1089 } else if (cmd_dir == DMA_TO_DEVICE) { 1090 data_direction = UTP_HOST_TO_DEVICE; 1091 *upiu_flags = UPIU_CMD_FLAGS_WRITE; 1092 } else { 1093 data_direction = UTP_NO_DATA_TRANSFER; 1094 *upiu_flags = UPIU_CMD_FLAGS_NONE; 1095 } 1096 1097 dword_0 = data_direction | (lrbp->command_type 1098 << UPIU_COMMAND_TYPE_OFFSET); 1099 if (lrbp->intr_cmd) 1100 dword_0 |= UTP_REQ_DESC_INT_CMD; 1101 1102 /* Transfer request descriptor header fields */ 1103 req_desc->header.dword_0 = cpu_to_le32(dword_0); 1104 1105 /* 1106 * assigning invalid value for command status. Controller 1107 * updates OCS on command completion, with the command 1108 * status 1109 */ 1110 req_desc->header.dword_2 = 1111 cpu_to_le32(OCS_INVALID_COMMAND_STATUS); 1112} 1113 1114/** 1115 * ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc, 1116 * for scsi commands 1117 * @lrbp - local reference block pointer 1118 * @upiu_flags - flags 1119 */ 1120static 1121void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u32 upiu_flags) 1122{ 1123 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr; 1124 1125 /* command descriptor fields */ 1126 ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD( 1127 UPIU_TRANSACTION_COMMAND, upiu_flags, 1128 lrbp->lun, lrbp->task_tag); 1129 ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD( 1130 UPIU_COMMAND_SET_TYPE_SCSI, 0, 0, 0); 1131 1132 /* Total EHS length and Data segment length will be zero */ 1133 ucd_req_ptr->header.dword_2 = 0; 1134 1135 ucd_req_ptr->sc.exp_data_transfer_len = 1136 cpu_to_be32(lrbp->cmd->sdb.length); 1137 1138 memcpy(ucd_req_ptr->sc.cdb, lrbp->cmd->cmnd, 1139 (min_t(unsigned short, lrbp->cmd->cmd_len, MAX_CDB_SIZE))); 1140} 1141 1142/** 1143 * ufshcd_prepare_utp_query_req_upiu() - fills the utp_transfer_req_desc, 1144 * for query requsts 1145 * @hba: UFS hba 1146 * @lrbp: local reference block pointer 1147 * @upiu_flags: flags 1148 */ 1149static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba, 1150 struct ufshcd_lrb *lrbp, u32 upiu_flags) 1151{ 1152 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr; 1153 struct ufs_query *query = &hba->dev_cmd.query; 1154 u16 len = be16_to_cpu(query->request.upiu_req.length); 1155 u8 *descp = (u8 *)lrbp->ucd_req_ptr + GENERAL_UPIU_REQUEST_SIZE; 1156 1157 /* Query request header */ 1158 ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD( 1159 UPIU_TRANSACTION_QUERY_REQ, upiu_flags, 1160 lrbp->lun, lrbp->task_tag); 1161 ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD( 1162 0, query->request.query_func, 0, 0); 1163 1164 /* Data segment length */ 1165 ucd_req_ptr->header.dword_2 = UPIU_HEADER_DWORD( 1166 0, 0, len >> 8, (u8)len); 1167 1168 /* Copy the Query Request buffer as is */ 1169 memcpy(&ucd_req_ptr->qr, &query->request.upiu_req, 1170 QUERY_OSF_SIZE); 1171 1172 /* Copy the Descriptor */ 1173 if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC) 1174 memcpy(descp, query->descriptor, len); 1175 1176} 1177 1178static inline void ufshcd_prepare_utp_nop_upiu(struct ufshcd_lrb *lrbp) 1179{ 1180 struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr; 1181 1182 memset(ucd_req_ptr, 0, sizeof(struct utp_upiu_req)); 1183 1184 /* command descriptor fields */ 1185 ucd_req_ptr->header.dword_0 = 1186 UPIU_HEADER_DWORD( 1187 UPIU_TRANSACTION_NOP_OUT, 0, 0, lrbp->task_tag); 1188} 1189 1190/** 1191 * ufshcd_compose_upiu - form UFS Protocol Information Unit(UPIU) 1192 * @hba - per adapter instance 1193 * @lrb - pointer to local reference block 1194 */ 1195static int ufshcd_compose_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) 1196{ 1197 u32 upiu_flags; 1198 int ret = 0; 1199 1200 switch (lrbp->command_type) { 1201 case UTP_CMD_TYPE_SCSI: 1202 if (likely(lrbp->cmd)) { 1203 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, 1204 lrbp->cmd->sc_data_direction); 1205 ufshcd_prepare_utp_scsi_cmd_upiu(lrbp, upiu_flags); 1206 } else { 1207 ret = -EINVAL; 1208 } 1209 break; 1210 case UTP_CMD_TYPE_DEV_MANAGE: 1211 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE); 1212 if (hba->dev_cmd.type == DEV_CMD_TYPE_QUERY) 1213 ufshcd_prepare_utp_query_req_upiu( 1214 hba, lrbp, upiu_flags); 1215 else if (hba->dev_cmd.type == DEV_CMD_TYPE_NOP) 1216 ufshcd_prepare_utp_nop_upiu(lrbp); 1217 else 1218 ret = -EINVAL; 1219 break; 1220 case UTP_CMD_TYPE_UFS: 1221 /* For UFS native command implementation */ 1222 ret = -ENOTSUPP; 1223 dev_err(hba->dev, "%s: UFS native command are not supported\n", 1224 __func__); 1225 break; 1226 default: 1227 ret = -ENOTSUPP; 1228 dev_err(hba->dev, "%s: unknown command type: 0x%x\n", 1229 __func__, lrbp->command_type); 1230 break; 1231 } /* end of switch */ 1232 1233 return ret; 1234} 1235 1236/* 1237 * ufshcd_scsi_to_upiu_lun - maps scsi LUN to UPIU LUN 1238 * @scsi_lun: scsi LUN id 1239 * 1240 * Returns UPIU LUN id 1241 */ 1242static inline u8 ufshcd_scsi_to_upiu_lun(unsigned int scsi_lun) 1243{ 1244 if (scsi_is_wlun(scsi_lun)) 1245 return (scsi_lun & UFS_UPIU_MAX_UNIT_NUM_ID) 1246 | UFS_UPIU_WLUN_ID; 1247 else 1248 return scsi_lun & UFS_UPIU_MAX_UNIT_NUM_ID; 1249} 1250 1251/** 1252 * ufshcd_upiu_wlun_to_scsi_wlun - maps UPIU W-LUN id to SCSI W-LUN ID 1253 * @scsi_lun: UPIU W-LUN id 1254 * 1255 * Returns SCSI W-LUN id 1256 */ 1257static inline u16 ufshcd_upiu_wlun_to_scsi_wlun(u8 upiu_wlun_id) 1258{ 1259 return (upiu_wlun_id & ~UFS_UPIU_WLUN_ID) | SCSI_W_LUN_BASE; 1260} 1261 1262/** 1263 * ufshcd_queuecommand - main entry point for SCSI requests 1264 * @cmd: command from SCSI Midlayer 1265 * @done: call back function 1266 * 1267 * Returns 0 for success, non-zero in case of failure 1268 */ 1269static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd) 1270{ 1271 struct ufshcd_lrb *lrbp; 1272 struct ufs_hba *hba; 1273 unsigned long flags; 1274 int tag; 1275 int err = 0; 1276 1277 hba = shost_priv(host); 1278 1279 tag = cmd->request->tag; 1280 1281 spin_lock_irqsave(hba->host->host_lock, flags); 1282 switch (hba->ufshcd_state) { 1283 case UFSHCD_STATE_OPERATIONAL: 1284 break; 1285 case UFSHCD_STATE_RESET: 1286 err = SCSI_MLQUEUE_HOST_BUSY; 1287 goto out_unlock; 1288 case UFSHCD_STATE_ERROR: 1289 set_host_byte(cmd, DID_ERROR); 1290 cmd->scsi_done(cmd); 1291 goto out_unlock; 1292 default: 1293 dev_WARN_ONCE(hba->dev, 1, "%s: invalid state %d\n", 1294 __func__, hba->ufshcd_state); 1295 set_host_byte(cmd, DID_BAD_TARGET); 1296 cmd->scsi_done(cmd); 1297 goto out_unlock; 1298 } 1299 spin_unlock_irqrestore(hba->host->host_lock, flags); 1300 1301 /* acquire the tag to make sure device cmds don't use it */ 1302 if (test_and_set_bit_lock(tag, &hba->lrb_in_use)) { 1303 /* 1304 * Dev manage command in progress, requeue the command. 1305 * Requeuing the command helps in cases where the request *may* 1306 * find different tag instead of waiting for dev manage command 1307 * completion. 1308 */ 1309 err = SCSI_MLQUEUE_HOST_BUSY; 1310 goto out; 1311 } 1312 1313 err = ufshcd_hold(hba, true); 1314 if (err) { 1315 err = SCSI_MLQUEUE_HOST_BUSY; 1316 clear_bit_unlock(tag, &hba->lrb_in_use); 1317 goto out; 1318 } 1319 WARN_ON(hba->clk_gating.state != CLKS_ON); 1320 1321 lrbp = &hba->lrb[tag]; 1322 1323 WARN_ON(lrbp->cmd); 1324 lrbp->cmd = cmd; 1325 lrbp->sense_bufflen = SCSI_SENSE_BUFFERSIZE; 1326 lrbp->sense_buffer = cmd->sense_buffer; 1327 lrbp->task_tag = tag; 1328 lrbp->lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun); 1329 lrbp->intr_cmd = false; 1330 lrbp->command_type = UTP_CMD_TYPE_SCSI; 1331 1332 /* form UPIU before issuing the command */ 1333 ufshcd_compose_upiu(hba, lrbp); 1334 err = ufshcd_map_sg(lrbp); 1335 if (err) { 1336 lrbp->cmd = NULL; 1337 clear_bit_unlock(tag, &hba->lrb_in_use); 1338 goto out; 1339 } 1340 1341 /* issue command to the controller */ 1342 spin_lock_irqsave(hba->host->host_lock, flags); 1343 ufshcd_send_command(hba, tag); 1344out_unlock: 1345 spin_unlock_irqrestore(hba->host->host_lock, flags); 1346out: 1347 return err; 1348} 1349 1350static int ufshcd_compose_dev_cmd(struct ufs_hba *hba, 1351 struct ufshcd_lrb *lrbp, enum dev_cmd_type cmd_type, int tag) 1352{ 1353 lrbp->cmd = NULL; 1354 lrbp->sense_bufflen = 0; 1355 lrbp->sense_buffer = NULL; 1356 lrbp->task_tag = tag; 1357 lrbp->lun = 0; /* device management cmd is not specific to any LUN */ 1358 lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE; 1359 lrbp->intr_cmd = true; /* No interrupt aggregation */ 1360 hba->dev_cmd.type = cmd_type; 1361 1362 return ufshcd_compose_upiu(hba, lrbp); 1363} 1364 1365static int 1366ufshcd_clear_cmd(struct ufs_hba *hba, int tag) 1367{ 1368 int err = 0; 1369 unsigned long flags; 1370 u32 mask = 1 << tag; 1371 1372 /* clear outstanding transaction before retry */ 1373 spin_lock_irqsave(hba->host->host_lock, flags); 1374 ufshcd_utrl_clear(hba, tag); 1375 spin_unlock_irqrestore(hba->host->host_lock, flags); 1376 1377 /* 1378 * wait for for h/w to clear corresponding bit in door-bell. 1379 * max. wait is 1 sec. 1380 */ 1381 err = ufshcd_wait_for_register(hba, 1382 REG_UTP_TRANSFER_REQ_DOOR_BELL, 1383 mask, ~mask, 1000, 1000); 1384 1385 return err; 1386} 1387 1388static int 1389ufshcd_check_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) 1390{ 1391 struct ufs_query_res *query_res = &hba->dev_cmd.query.response; 1392 1393 /* Get the UPIU response */ 1394 query_res->response = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr) >> 1395 UPIU_RSP_CODE_OFFSET; 1396 return query_res->response; 1397} 1398 1399/** 1400 * ufshcd_dev_cmd_completion() - handles device management command responses 1401 * @hba: per adapter instance 1402 * @lrbp: pointer to local reference block 1403 */ 1404static int 1405ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) 1406{ 1407 int resp; 1408 int err = 0; 1409 1410 resp = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr); 1411 1412 switch (resp) { 1413 case UPIU_TRANSACTION_NOP_IN: 1414 if (hba->dev_cmd.type != DEV_CMD_TYPE_NOP) { 1415 err = -EINVAL; 1416 dev_err(hba->dev, "%s: unexpected response %x\n", 1417 __func__, resp); 1418 } 1419 break; 1420 case UPIU_TRANSACTION_QUERY_RSP: 1421 err = ufshcd_check_query_response(hba, lrbp); 1422 if (!err) 1423 err = ufshcd_copy_query_response(hba, lrbp); 1424 break; 1425 case UPIU_TRANSACTION_REJECT_UPIU: 1426 /* TODO: handle Reject UPIU Response */ 1427 err = -EPERM; 1428 dev_err(hba->dev, "%s: Reject UPIU not fully implemented\n", 1429 __func__); 1430 break; 1431 default: 1432 err = -EINVAL; 1433 dev_err(hba->dev, "%s: Invalid device management cmd response: %x\n", 1434 __func__, resp); 1435 break; 1436 } 1437 1438 return err; 1439} 1440 1441static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba, 1442 struct ufshcd_lrb *lrbp, int max_timeout) 1443{ 1444 int err = 0; 1445 unsigned long time_left; 1446 unsigned long flags; 1447 1448 time_left = wait_for_completion_timeout(hba->dev_cmd.complete, 1449 msecs_to_jiffies(max_timeout)); 1450 1451 spin_lock_irqsave(hba->host->host_lock, flags); 1452 hba->dev_cmd.complete = NULL; 1453 if (likely(time_left)) { 1454 err = ufshcd_get_tr_ocs(lrbp); 1455 if (!err) 1456 err = ufshcd_dev_cmd_completion(hba, lrbp); 1457 } 1458 spin_unlock_irqrestore(hba->host->host_lock, flags); 1459 1460 if (!time_left) { 1461 err = -ETIMEDOUT; 1462 if (!ufshcd_clear_cmd(hba, lrbp->task_tag)) 1463 /* sucessfully cleared the command, retry if needed */ 1464 err = -EAGAIN; 1465 } 1466 1467 return err; 1468} 1469 1470/** 1471 * ufshcd_get_dev_cmd_tag - Get device management command tag 1472 * @hba: per-adapter instance 1473 * @tag: pointer to variable with available slot value 1474 * 1475 * Get a free slot and lock it until device management command 1476 * completes. 1477 * 1478 * Returns false if free slot is unavailable for locking, else 1479 * return true with tag value in @tag. 1480 */ 1481static bool ufshcd_get_dev_cmd_tag(struct ufs_hba *hba, int *tag_out) 1482{ 1483 int tag; 1484 bool ret = false; 1485 unsigned long tmp; 1486 1487 if (!tag_out) 1488 goto out; 1489 1490 do { 1491 tmp = ~hba->lrb_in_use; 1492 tag = find_last_bit(&tmp, hba->nutrs); 1493 if (tag >= hba->nutrs) 1494 goto out; 1495 } while (test_and_set_bit_lock(tag, &hba->lrb_in_use)); 1496 1497 *tag_out = tag; 1498 ret = true; 1499out: 1500 return ret; 1501} 1502 1503static inline void ufshcd_put_dev_cmd_tag(struct ufs_hba *hba, int tag) 1504{ 1505 clear_bit_unlock(tag, &hba->lrb_in_use); 1506} 1507 1508/** 1509 * ufshcd_exec_dev_cmd - API for sending device management requests 1510 * @hba - UFS hba 1511 * @cmd_type - specifies the type (NOP, Query...) 1512 * @timeout - time in seconds 1513 * 1514 * NOTE: Since there is only one available tag for device management commands, 1515 * it is expected you hold the hba->dev_cmd.lock mutex. 1516 */ 1517static int ufshcd_exec_dev_cmd(struct ufs_hba *hba, 1518 enum dev_cmd_type cmd_type, int timeout) 1519{ 1520 struct ufshcd_lrb *lrbp; 1521 int err; 1522 int tag; 1523 struct completion wait; 1524 unsigned long flags; 1525 1526 /* 1527 * Get free slot, sleep if slots are unavailable. 1528 * Even though we use wait_event() which sleeps indefinitely, 1529 * the maximum wait time is bounded by SCSI request timeout. 1530 */ 1531 wait_event(hba->dev_cmd.tag_wq, ufshcd_get_dev_cmd_tag(hba, &tag)); 1532 1533 init_completion(&wait); 1534 lrbp = &hba->lrb[tag]; 1535 WARN_ON(lrbp->cmd); 1536 err = ufshcd_compose_dev_cmd(hba, lrbp, cmd_type, tag); 1537 if (unlikely(err)) 1538 goto out_put_tag; 1539 1540 hba->dev_cmd.complete = &wait; 1541 1542 spin_lock_irqsave(hba->host->host_lock, flags); 1543 ufshcd_send_command(hba, tag); 1544 spin_unlock_irqrestore(hba->host->host_lock, flags); 1545 1546 err = ufshcd_wait_for_dev_cmd(hba, lrbp, timeout); 1547 1548out_put_tag: 1549 ufshcd_put_dev_cmd_tag(hba, tag); 1550 wake_up(&hba->dev_cmd.tag_wq); 1551 return err; 1552} 1553 1554/** 1555 * ufshcd_init_query() - init the query response and request parameters 1556 * @hba: per-adapter instance 1557 * @request: address of the request pointer to be initialized 1558 * @response: address of the response pointer to be initialized 1559 * @opcode: operation to perform 1560 * @idn: flag idn to access 1561 * @index: LU number to access 1562 * @selector: query/flag/descriptor further identification 1563 */ 1564static inline void ufshcd_init_query(struct ufs_hba *hba, 1565 struct ufs_query_req **request, struct ufs_query_res **response, 1566 enum query_opcode opcode, u8 idn, u8 index, u8 selector) 1567{ 1568 *request = &hba->dev_cmd.query.request; 1569 *response = &hba->dev_cmd.query.response; 1570 memset(*request, 0, sizeof(struct ufs_query_req)); 1571 memset(*response, 0, sizeof(struct ufs_query_res)); 1572 (*request)->upiu_req.opcode = opcode; 1573 (*request)->upiu_req.idn = idn; 1574 (*request)->upiu_req.index = index; 1575 (*request)->upiu_req.selector = selector; 1576} 1577 1578/** 1579 * ufshcd_query_flag() - API function for sending flag query requests 1580 * hba: per-adapter instance 1581 * query_opcode: flag query to perform 1582 * idn: flag idn to access 1583 * flag_res: the flag value after the query request completes 1584 * 1585 * Returns 0 for success, non-zero in case of failure 1586 */ 1587static int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode, 1588 enum flag_idn idn, bool *flag_res) 1589{ 1590 struct ufs_query_req *request = NULL; 1591 struct ufs_query_res *response = NULL; 1592 int err, index = 0, selector = 0; 1593 1594 BUG_ON(!hba); 1595 1596 ufshcd_hold(hba, false); 1597 mutex_lock(&hba->dev_cmd.lock); 1598 ufshcd_init_query(hba, &request, &response, opcode, idn, index, 1599 selector); 1600 1601 switch (opcode) { 1602 case UPIU_QUERY_OPCODE_SET_FLAG: 1603 case UPIU_QUERY_OPCODE_CLEAR_FLAG: 1604 case UPIU_QUERY_OPCODE_TOGGLE_FLAG: 1605 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST; 1606 break; 1607 case UPIU_QUERY_OPCODE_READ_FLAG: 1608 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST; 1609 if (!flag_res) { 1610 /* No dummy reads */ 1611 dev_err(hba->dev, "%s: Invalid argument for read request\n", 1612 __func__); 1613 err = -EINVAL; 1614 goto out_unlock; 1615 } 1616 break; 1617 default: 1618 dev_err(hba->dev, 1619 "%s: Expected query flag opcode but got = %d\n", 1620 __func__, opcode); 1621 err = -EINVAL; 1622 goto out_unlock; 1623 } 1624 1625 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT); 1626 1627 if (err) { 1628 dev_err(hba->dev, 1629 "%s: Sending flag query for idn %d failed, err = %d\n", 1630 __func__, idn, err); 1631 goto out_unlock; 1632 } 1633 1634 if (flag_res) 1635 *flag_res = (be32_to_cpu(response->upiu_res.value) & 1636 MASK_QUERY_UPIU_FLAG_LOC) & 0x1; 1637 1638out_unlock: 1639 mutex_unlock(&hba->dev_cmd.lock); 1640 ufshcd_release(hba); 1641 return err; 1642} 1643 1644/** 1645 * ufshcd_query_attr - API function for sending attribute requests 1646 * hba: per-adapter instance 1647 * opcode: attribute opcode 1648 * idn: attribute idn to access 1649 * index: index field 1650 * selector: selector field 1651 * attr_val: the attribute value after the query request completes 1652 * 1653 * Returns 0 for success, non-zero in case of failure 1654*/ 1655static int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode, 1656 enum attr_idn idn, u8 index, u8 selector, u32 *attr_val) 1657{ 1658 struct ufs_query_req *request = NULL; 1659 struct ufs_query_res *response = NULL; 1660 int err; 1661 1662 BUG_ON(!hba); 1663 1664 ufshcd_hold(hba, false); 1665 if (!attr_val) { 1666 dev_err(hba->dev, "%s: attribute value required for opcode 0x%x\n", 1667 __func__, opcode); 1668 err = -EINVAL; 1669 goto out; 1670 } 1671 1672 mutex_lock(&hba->dev_cmd.lock); 1673 ufshcd_init_query(hba, &request, &response, opcode, idn, index, 1674 selector); 1675 1676 switch (opcode) { 1677 case UPIU_QUERY_OPCODE_WRITE_ATTR: 1678 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST; 1679 request->upiu_req.value = cpu_to_be32(*attr_val); 1680 break; 1681 case UPIU_QUERY_OPCODE_READ_ATTR: 1682 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST; 1683 break; 1684 default: 1685 dev_err(hba->dev, "%s: Expected query attr opcode but got = 0x%.2x\n", 1686 __func__, opcode); 1687 err = -EINVAL; 1688 goto out_unlock; 1689 } 1690 1691 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT); 1692 1693 if (err) { 1694 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, err = %d\n", 1695 __func__, opcode, idn, err); 1696 goto out_unlock; 1697 } 1698 1699 *attr_val = be32_to_cpu(response->upiu_res.value); 1700 1701out_unlock: 1702 mutex_unlock(&hba->dev_cmd.lock); 1703out: 1704 ufshcd_release(hba); 1705 return err; 1706} 1707 1708/** 1709 * ufshcd_query_descriptor - API function for sending descriptor requests 1710 * hba: per-adapter instance 1711 * opcode: attribute opcode 1712 * idn: attribute idn to access 1713 * index: index field 1714 * selector: selector field 1715 * desc_buf: the buffer that contains the descriptor 1716 * buf_len: length parameter passed to the device 1717 * 1718 * Returns 0 for success, non-zero in case of failure. 1719 * The buf_len parameter will contain, on return, the length parameter 1720 * received on the response. 1721 */ 1722static int ufshcd_query_descriptor(struct ufs_hba *hba, 1723 enum query_opcode opcode, enum desc_idn idn, u8 index, 1724 u8 selector, u8 *desc_buf, int *buf_len) 1725{ 1726 struct ufs_query_req *request = NULL; 1727 struct ufs_query_res *response = NULL; 1728 int err; 1729 1730 BUG_ON(!hba); 1731 1732 ufshcd_hold(hba, false); 1733 if (!desc_buf) { 1734 dev_err(hba->dev, "%s: descriptor buffer required for opcode 0x%x\n", 1735 __func__, opcode); 1736 err = -EINVAL; 1737 goto out; 1738 } 1739 1740 if (*buf_len <= QUERY_DESC_MIN_SIZE || *buf_len > QUERY_DESC_MAX_SIZE) { 1741 dev_err(hba->dev, "%s: descriptor buffer size (%d) is out of range\n", 1742 __func__, *buf_len); 1743 err = -EINVAL; 1744 goto out; 1745 } 1746 1747 mutex_lock(&hba->dev_cmd.lock); 1748 ufshcd_init_query(hba, &request, &response, opcode, idn, index, 1749 selector); 1750 hba->dev_cmd.query.descriptor = desc_buf; 1751 request->upiu_req.length = cpu_to_be16(*buf_len); 1752 1753 switch (opcode) { 1754 case UPIU_QUERY_OPCODE_WRITE_DESC: 1755 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST; 1756 break; 1757 case UPIU_QUERY_OPCODE_READ_DESC: 1758 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST; 1759 break; 1760 default: 1761 dev_err(hba->dev, 1762 "%s: Expected query descriptor opcode but got = 0x%.2x\n", 1763 __func__, opcode); 1764 err = -EINVAL; 1765 goto out_unlock; 1766 } 1767 1768 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT); 1769 1770 if (err) { 1771 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, err = %d\n", 1772 __func__, opcode, idn, err); 1773 goto out_unlock; 1774 } 1775 1776 hba->dev_cmd.query.descriptor = NULL; 1777 *buf_len = be16_to_cpu(response->upiu_res.length); 1778 1779out_unlock: 1780 mutex_unlock(&hba->dev_cmd.lock); 1781out: 1782 ufshcd_release(hba); 1783 return err; 1784} 1785 1786/** 1787 * ufshcd_read_desc_param - read the specified descriptor parameter 1788 * @hba: Pointer to adapter instance 1789 * @desc_id: descriptor idn value 1790 * @desc_index: descriptor index 1791 * @param_offset: offset of the parameter to read 1792 * @param_read_buf: pointer to buffer where parameter would be read 1793 * @param_size: sizeof(param_read_buf) 1794 * 1795 * Return 0 in case of success, non-zero otherwise 1796 */ 1797static int ufshcd_read_desc_param(struct ufs_hba *hba, 1798 enum desc_idn desc_id, 1799 int desc_index, 1800 u32 param_offset, 1801 u8 *param_read_buf, 1802 u32 param_size) 1803{ 1804 int ret; 1805 u8 *desc_buf; 1806 u32 buff_len; 1807 bool is_kmalloc = true; 1808 1809 /* safety checks */ 1810 if (desc_id >= QUERY_DESC_IDN_MAX) 1811 return -EINVAL; 1812 1813 buff_len = ufs_query_desc_max_size[desc_id]; 1814 if ((param_offset + param_size) > buff_len) 1815 return -EINVAL; 1816 1817 if (!param_offset && (param_size == buff_len)) { 1818 /* memory space already available to hold full descriptor */ 1819 desc_buf = param_read_buf; 1820 is_kmalloc = false; 1821 } else { 1822 /* allocate memory to hold full descriptor */ 1823 desc_buf = kmalloc(buff_len, GFP_KERNEL); 1824 if (!desc_buf) 1825 return -ENOMEM; 1826 } 1827 1828 ret = ufshcd_query_descriptor(hba, UPIU_QUERY_OPCODE_READ_DESC, 1829 desc_id, desc_index, 0, desc_buf, 1830 &buff_len); 1831 1832 if (ret || (buff_len < ufs_query_desc_max_size[desc_id]) || 1833 (desc_buf[QUERY_DESC_LENGTH_OFFSET] != 1834 ufs_query_desc_max_size[desc_id]) 1835 || (desc_buf[QUERY_DESC_DESC_TYPE_OFFSET] != desc_id)) { 1836 dev_err(hba->dev, "%s: Failed reading descriptor. desc_id %d param_offset %d buff_len %d ret %d", 1837 __func__, desc_id, param_offset, buff_len, ret); 1838 if (!ret) 1839 ret = -EINVAL; 1840 1841 goto out; 1842 } 1843 1844 if (is_kmalloc) 1845 memcpy(param_read_buf, &desc_buf[param_offset], param_size); 1846out: 1847 if (is_kmalloc) 1848 kfree(desc_buf); 1849 return ret; 1850} 1851 1852static inline int ufshcd_read_desc(struct ufs_hba *hba, 1853 enum desc_idn desc_id, 1854 int desc_index, 1855 u8 *buf, 1856 u32 size) 1857{ 1858 return ufshcd_read_desc_param(hba, desc_id, desc_index, 0, buf, size); 1859} 1860 1861static inline int ufshcd_read_power_desc(struct ufs_hba *hba, 1862 u8 *buf, 1863 u32 size) 1864{ 1865 return ufshcd_read_desc(hba, QUERY_DESC_IDN_POWER, 0, buf, size); 1866} 1867 1868/** 1869 * ufshcd_read_unit_desc_param - read the specified unit descriptor parameter 1870 * @hba: Pointer to adapter instance 1871 * @lun: lun id 1872 * @param_offset: offset of the parameter to read 1873 * @param_read_buf: pointer to buffer where parameter would be read 1874 * @param_size: sizeof(param_read_buf) 1875 * 1876 * Return 0 in case of success, non-zero otherwise 1877 */ 1878static inline int ufshcd_read_unit_desc_param(struct ufs_hba *hba, 1879 int lun, 1880 enum unit_desc_param param_offset, 1881 u8 *param_read_buf, 1882 u32 param_size) 1883{ 1884 /* 1885 * Unit descriptors are only available for general purpose LUs (LUN id 1886 * from 0 to 7) and RPMB Well known LU. 1887 */ 1888 if (lun != UFS_UPIU_RPMB_WLUN && (lun >= UFS_UPIU_MAX_GENERAL_LUN)) 1889 return -EOPNOTSUPP; 1890 1891 return ufshcd_read_desc_param(hba, QUERY_DESC_IDN_UNIT, lun, 1892 param_offset, param_read_buf, param_size); 1893} 1894 1895/** 1896 * ufshcd_memory_alloc - allocate memory for host memory space data structures 1897 * @hba: per adapter instance 1898 * 1899 * 1. Allocate DMA memory for Command Descriptor array 1900 * Each command descriptor consist of Command UPIU, Response UPIU and PRDT 1901 * 2. Allocate DMA memory for UTP Transfer Request Descriptor List (UTRDL). 1902 * 3. Allocate DMA memory for UTP Task Management Request Descriptor List 1903 * (UTMRDL) 1904 * 4. Allocate memory for local reference block(lrb). 1905 * 1906 * Returns 0 for success, non-zero in case of failure 1907 */ 1908static int ufshcd_memory_alloc(struct ufs_hba *hba) 1909{ 1910 size_t utmrdl_size, utrdl_size, ucdl_size; 1911 1912 /* Allocate memory for UTP command descriptors */ 1913 ucdl_size = (sizeof(struct utp_transfer_cmd_desc) * hba->nutrs); 1914 hba->ucdl_base_addr = dmam_alloc_coherent(hba->dev, 1915 ucdl_size, 1916 &hba->ucdl_dma_addr, 1917 GFP_KERNEL); 1918 1919 /* 1920 * UFSHCI requires UTP command descriptor to be 128 byte aligned. 1921 * make sure hba->ucdl_dma_addr is aligned to PAGE_SIZE 1922 * if hba->ucdl_dma_addr is aligned to PAGE_SIZE, then it will 1923 * be aligned to 128 bytes as well 1924 */ 1925 if (!hba->ucdl_base_addr || 1926 WARN_ON(hba->ucdl_dma_addr & (PAGE_SIZE - 1))) { 1927 dev_err(hba->dev, 1928 "Command Descriptor Memory allocation failed\n"); 1929 goto out; 1930 } 1931 1932 /* 1933 * Allocate memory for UTP Transfer descriptors 1934 * UFSHCI requires 1024 byte alignment of UTRD 1935 */ 1936 utrdl_size = (sizeof(struct utp_transfer_req_desc) * hba->nutrs); 1937 hba->utrdl_base_addr = dmam_alloc_coherent(hba->dev, 1938 utrdl_size, 1939 &hba->utrdl_dma_addr, 1940 GFP_KERNEL); 1941 if (!hba->utrdl_base_addr || 1942 WARN_ON(hba->utrdl_dma_addr & (PAGE_SIZE - 1))) { 1943 dev_err(hba->dev, 1944 "Transfer Descriptor Memory allocation failed\n"); 1945 goto out; 1946 } 1947 1948 /* 1949 * Allocate memory for UTP Task Management descriptors 1950 * UFSHCI requires 1024 byte alignment of UTMRD 1951 */ 1952 utmrdl_size = sizeof(struct utp_task_req_desc) * hba->nutmrs; 1953 hba->utmrdl_base_addr = dmam_alloc_coherent(hba->dev, 1954 utmrdl_size, 1955 &hba->utmrdl_dma_addr, 1956 GFP_KERNEL); 1957 if (!hba->utmrdl_base_addr || 1958 WARN_ON(hba->utmrdl_dma_addr & (PAGE_SIZE - 1))) { 1959 dev_err(hba->dev, 1960 "Task Management Descriptor Memory allocation failed\n"); 1961 goto out; 1962 } 1963 1964 /* Allocate memory for local reference block */ 1965 hba->lrb = devm_kzalloc(hba->dev, 1966 hba->nutrs * sizeof(struct ufshcd_lrb), 1967 GFP_KERNEL); 1968 if (!hba->lrb) { 1969 dev_err(hba->dev, "LRB Memory allocation failed\n"); 1970 goto out; 1971 } 1972 return 0; 1973out: 1974 return -ENOMEM; 1975} 1976 1977/** 1978 * ufshcd_host_memory_configure - configure local reference block with 1979 * memory offsets 1980 * @hba: per adapter instance 1981 * 1982 * Configure Host memory space 1983 * 1. Update Corresponding UTRD.UCDBA and UTRD.UCDBAU with UCD DMA 1984 * address. 1985 * 2. Update each UTRD with Response UPIU offset, Response UPIU length 1986 * and PRDT offset. 1987 * 3. Save the corresponding addresses of UTRD, UCD.CMD, UCD.RSP and UCD.PRDT 1988 * into local reference block. 1989 */ 1990static void ufshcd_host_memory_configure(struct ufs_hba *hba) 1991{ 1992 struct utp_transfer_cmd_desc *cmd_descp; 1993 struct utp_transfer_req_desc *utrdlp; 1994 dma_addr_t cmd_desc_dma_addr; 1995 dma_addr_t cmd_desc_element_addr; 1996 u16 response_offset; 1997 u16 prdt_offset; 1998 int cmd_desc_size; 1999 int i; 2000 2001 utrdlp = hba->utrdl_base_addr; 2002 cmd_descp = hba->ucdl_base_addr; 2003 2004 response_offset = 2005 offsetof(struct utp_transfer_cmd_desc, response_upiu); 2006 prdt_offset = 2007 offsetof(struct utp_transfer_cmd_desc, prd_table); 2008 2009 cmd_desc_size = sizeof(struct utp_transfer_cmd_desc); 2010 cmd_desc_dma_addr = hba->ucdl_dma_addr; 2011 2012 for (i = 0; i < hba->nutrs; i++) { 2013 /* Configure UTRD with command descriptor base address */ 2014 cmd_desc_element_addr = 2015 (cmd_desc_dma_addr + (cmd_desc_size * i)); 2016 utrdlp[i].command_desc_base_addr_lo = 2017 cpu_to_le32(lower_32_bits(cmd_desc_element_addr)); 2018 utrdlp[i].command_desc_base_addr_hi = 2019 cpu_to_le32(upper_32_bits(cmd_desc_element_addr)); 2020 2021 /* Response upiu and prdt offset should be in double words */ 2022 utrdlp[i].response_upiu_offset = 2023 cpu_to_le16((response_offset >> 2)); 2024 utrdlp[i].prd_table_offset = 2025 cpu_to_le16((prdt_offset >> 2)); 2026 utrdlp[i].response_upiu_length = 2027 cpu_to_le16(ALIGNED_UPIU_SIZE >> 2); 2028 2029 hba->lrb[i].utr_descriptor_ptr = (utrdlp + i); 2030 hba->lrb[i].ucd_req_ptr = 2031 (struct utp_upiu_req *)(cmd_descp + i); 2032 hba->lrb[i].ucd_rsp_ptr = 2033 (struct utp_upiu_rsp *)cmd_descp[i].response_upiu; 2034 hba->lrb[i].ucd_prdt_ptr = 2035 (struct ufshcd_sg_entry *)cmd_descp[i].prd_table; 2036 } 2037} 2038 2039/** 2040 * ufshcd_dme_link_startup - Notify Unipro to perform link startup 2041 * @hba: per adapter instance 2042 * 2043 * UIC_CMD_DME_LINK_STARTUP command must be issued to Unipro layer, 2044 * in order to initialize the Unipro link startup procedure. 2045 * Once the Unipro links are up, the device connected to the controller 2046 * is detected. 2047 * 2048 * Returns 0 on success, non-zero value on failure 2049 */ 2050static int ufshcd_dme_link_startup(struct ufs_hba *hba) 2051{ 2052 struct uic_command uic_cmd = {0}; 2053 int ret; 2054 2055 uic_cmd.command = UIC_CMD_DME_LINK_STARTUP; 2056 2057 ret = ufshcd_send_uic_cmd(hba, &uic_cmd); 2058 if (ret) 2059 dev_err(hba->dev, 2060 "dme-link-startup: error code %d\n", ret); 2061 return ret; 2062} 2063 2064static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba) 2065{ 2066 #define MIN_DELAY_BEFORE_DME_CMDS_US 1000 2067 unsigned long min_sleep_time_us; 2068 2069 if (!(hba->quirks & UFSHCD_QUIRK_DELAY_BEFORE_DME_CMDS)) 2070 return; 2071 2072 /* 2073 * last_dme_cmd_tstamp will be 0 only for 1st call to 2074 * this function 2075 */ 2076 if (unlikely(!ktime_to_us(hba->last_dme_cmd_tstamp))) { 2077 min_sleep_time_us = MIN_DELAY_BEFORE_DME_CMDS_US; 2078 } else { 2079 unsigned long delta = 2080 (unsigned long) ktime_to_us( 2081 ktime_sub(ktime_get(), 2082 hba->last_dme_cmd_tstamp)); 2083 2084 if (delta < MIN_DELAY_BEFORE_DME_CMDS_US) 2085 min_sleep_time_us = 2086 MIN_DELAY_BEFORE_DME_CMDS_US - delta; 2087 else 2088 return; /* no more delay required */ 2089 } 2090 2091 /* allow sleep for extra 50us if needed */ 2092 usleep_range(min_sleep_time_us, min_sleep_time_us + 50); 2093} 2094 2095/** 2096 * ufshcd_dme_set_attr - UIC command for DME_SET, DME_PEER_SET 2097 * @hba: per adapter instance 2098 * @attr_sel: uic command argument1 2099 * @attr_set: attribute set type as uic command argument2 2100 * @mib_val: setting value as uic command argument3 2101 * @peer: indicate whether peer or local 2102 * 2103 * Returns 0 on success, non-zero value on failure 2104 */ 2105int ufshcd_dme_set_attr(struct ufs_hba *hba, u32 attr_sel, 2106 u8 attr_set, u32 mib_val, u8 peer) 2107{ 2108 struct uic_command uic_cmd = {0}; 2109 static const char *const action[] = { 2110 "dme-set", 2111 "dme-peer-set" 2112 }; 2113 const char *set = action[!!peer]; 2114 int ret; 2115 2116 uic_cmd.command = peer ? 2117 UIC_CMD_DME_PEER_SET : UIC_CMD_DME_SET; 2118 uic_cmd.argument1 = attr_sel; 2119 uic_cmd.argument2 = UIC_ARG_ATTR_TYPE(attr_set); 2120 uic_cmd.argument3 = mib_val; 2121 2122 ret = ufshcd_send_uic_cmd(hba, &uic_cmd); 2123 if (ret) 2124 dev_err(hba->dev, "%s: attr-id 0x%x val 0x%x error code %d\n", 2125 set, UIC_GET_ATTR_ID(attr_sel), mib_val, ret); 2126 2127 return ret; 2128} 2129EXPORT_SYMBOL_GPL(ufshcd_dme_set_attr); 2130 2131/** 2132 * ufshcd_dme_get_attr - UIC command for DME_GET, DME_PEER_GET 2133 * @hba: per adapter instance 2134 * @attr_sel: uic command argument1 2135 * @mib_val: the value of the attribute as returned by the UIC command 2136 * @peer: indicate whether peer or local 2137 * 2138 * Returns 0 on success, non-zero value on failure 2139 */ 2140int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel, 2141 u32 *mib_val, u8 peer) 2142{ 2143 struct uic_command uic_cmd = {0}; 2144 static const char *const action[] = { 2145 "dme-get", 2146 "dme-peer-get" 2147 }; 2148 const char *get = action[!!peer]; 2149 int ret; 2150 2151 uic_cmd.command = peer ? 2152 UIC_CMD_DME_PEER_GET : UIC_CMD_DME_GET; 2153 uic_cmd.argument1 = attr_sel; 2154 2155 ret = ufshcd_send_uic_cmd(hba, &uic_cmd); 2156 if (ret) { 2157 dev_err(hba->dev, "%s: attr-id 0x%x error code %d\n", 2158 get, UIC_GET_ATTR_ID(attr_sel), ret); 2159 goto out; 2160 } 2161 2162 if (mib_val) 2163 *mib_val = uic_cmd.argument3; 2164out: 2165 return ret; 2166} 2167EXPORT_SYMBOL_GPL(ufshcd_dme_get_attr); 2168 2169/** 2170 * ufshcd_uic_pwr_ctrl - executes UIC commands (which affects the link power 2171 * state) and waits for it to take effect. 2172 * 2173 * @hba: per adapter instance 2174 * @cmd: UIC command to execute 2175 * 2176 * DME operations like DME_SET(PA_PWRMODE), DME_HIBERNATE_ENTER & 2177 * DME_HIBERNATE_EXIT commands take some time to take its effect on both host 2178 * and device UniPro link and hence it's final completion would be indicated by 2179 * dedicated status bits in Interrupt Status register (UPMS, UHES, UHXS) in 2180 * addition to normal UIC command completion Status (UCCS). This function only 2181 * returns after the relevant status bits indicate the completion. 2182 * 2183 * Returns 0 on success, non-zero value on failure 2184 */ 2185static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd) 2186{ 2187 struct completion uic_async_done; 2188 unsigned long flags; 2189 u8 status; 2190 int ret; 2191 2192 mutex_lock(&hba->uic_cmd_mutex); 2193 init_completion(&uic_async_done); 2194 ufshcd_add_delay_before_dme_cmd(hba); 2195 2196 spin_lock_irqsave(hba->host->host_lock, flags); 2197 hba->uic_async_done = &uic_async_done; 2198 ret = __ufshcd_send_uic_cmd(hba, cmd); 2199 spin_unlock_irqrestore(hba->host->host_lock, flags); 2200 if (ret) { 2201 dev_err(hba->dev, 2202 "pwr ctrl cmd 0x%x with mode 0x%x uic error %d\n", 2203 cmd->command, cmd->argument3, ret); 2204 goto out; 2205 } 2206 ret = ufshcd_wait_for_uic_cmd(hba, cmd); 2207 if (ret) { 2208 dev_err(hba->dev, 2209 "pwr ctrl cmd 0x%x with mode 0x%x uic error %d\n", 2210 cmd->command, cmd->argument3, ret); 2211 goto out; 2212 } 2213 2214 if (!wait_for_completion_timeout(hba->uic_async_done, 2215 msecs_to_jiffies(UIC_CMD_TIMEOUT))) { 2216 dev_err(hba->dev, 2217 "pwr ctrl cmd 0x%x with mode 0x%x completion timeout\n", 2218 cmd->command, cmd->argument3); 2219 ret = -ETIMEDOUT; 2220 goto out; 2221 } 2222 2223 status = ufshcd_get_upmcrs(hba); 2224 if (status != PWR_LOCAL) { 2225 dev_err(hba->dev, 2226 "pwr ctrl cmd 0x%0x failed, host umpcrs:0x%x\n", 2227 cmd->command, status); 2228 ret = (status != PWR_OK) ? status : -1; 2229 } 2230out: 2231 spin_lock_irqsave(hba->host->host_lock, flags); 2232 hba->uic_async_done = NULL; 2233 spin_unlock_irqrestore(hba->host->host_lock, flags); 2234 mutex_unlock(&hba->uic_cmd_mutex); 2235 2236 return ret; 2237} 2238 2239/** 2240 * ufshcd_uic_change_pwr_mode - Perform the UIC power mode chage 2241 * using DME_SET primitives. 2242 * @hba: per adapter instance 2243 * @mode: powr mode value 2244 * 2245 * Returns 0 on success, non-zero value on failure 2246 */ 2247static int ufshcd_uic_change_pwr_mode(struct ufs_hba *hba, u8 mode) 2248{ 2249 struct uic_command uic_cmd = {0}; 2250 int ret; 2251 2252 uic_cmd.command = UIC_CMD_DME_SET; 2253 uic_cmd.argument1 = UIC_ARG_MIB(PA_PWRMODE); 2254 uic_cmd.argument3 = mode; 2255 ufshcd_hold(hba, false); 2256 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd); 2257 ufshcd_release(hba); 2258 2259 return ret; 2260} 2261 2262static int ufshcd_uic_hibern8_enter(struct ufs_hba *hba) 2263{ 2264 struct uic_command uic_cmd = {0}; 2265 2266 uic_cmd.command = UIC_CMD_DME_HIBER_ENTER; 2267 2268 return ufshcd_uic_pwr_ctrl(hba, &uic_cmd); 2269} 2270 2271static int ufshcd_uic_hibern8_exit(struct ufs_hba *hba) 2272{ 2273 struct uic_command uic_cmd = {0}; 2274 int ret; 2275 2276 uic_cmd.command = UIC_CMD_DME_HIBER_EXIT; 2277 ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd); 2278 if (ret) { 2279 ufshcd_set_link_off(hba); 2280 ret = ufshcd_host_reset_and_restore(hba); 2281 } 2282 2283 return ret; 2284} 2285 2286 /** 2287 * ufshcd_init_pwr_info - setting the POR (power on reset) 2288 * values in hba power info 2289 * @hba: per-adapter instance 2290 */ 2291static void ufshcd_init_pwr_info(struct ufs_hba *hba) 2292{ 2293 hba->pwr_info.gear_rx = UFS_PWM_G1; 2294 hba->pwr_info.gear_tx = UFS_PWM_G1; 2295 hba->pwr_info.lane_rx = 1; 2296 hba->pwr_info.lane_tx = 1; 2297 hba->pwr_info.pwr_rx = SLOWAUTO_MODE; 2298 hba->pwr_info.pwr_tx = SLOWAUTO_MODE; 2299 hba->pwr_info.hs_rate = 0; 2300} 2301 2302/** 2303 * ufshcd_get_max_pwr_mode - reads the max power mode negotiated with device 2304 * @hba: per-adapter instance 2305 */ 2306static int ufshcd_get_max_pwr_mode(struct ufs_hba *hba) 2307{ 2308 struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info; 2309 2310 if (hba->max_pwr_info.is_valid) 2311 return 0; 2312 2313 pwr_info->pwr_tx = FASTAUTO_MODE; 2314 pwr_info->pwr_rx = FASTAUTO_MODE; 2315 pwr_info->hs_rate = PA_HS_MODE_B; 2316 2317 /* Get the connected lane count */ 2318 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDRXDATALANES), 2319 &pwr_info->lane_rx); 2320 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES), 2321 &pwr_info->lane_tx); 2322 2323 if (!pwr_info->lane_rx || !pwr_info->lane_tx) { 2324 dev_err(hba->dev, "%s: invalid connected lanes value. rx=%d, tx=%d\n", 2325 __func__, 2326 pwr_info->lane_rx, 2327 pwr_info->lane_tx); 2328 return -EINVAL; 2329 } 2330 2331 /* 2332 * First, get the maximum gears of HS speed. 2333 * If a zero value, it means there is no HSGEAR capability. 2334 * Then, get the maximum gears of PWM speed. 2335 */ 2336 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR), &pwr_info->gear_rx); 2337 if (!pwr_info->gear_rx) { 2338 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR), 2339 &pwr_info->gear_rx); 2340 if (!pwr_info->gear_rx) { 2341 dev_err(hba->dev, "%s: invalid max pwm rx gear read = %d\n", 2342 __func__, pwr_info->gear_rx); 2343 return -EINVAL; 2344 } 2345 pwr_info->pwr_rx = SLOWAUTO_MODE; 2346 } 2347 2348 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR), 2349 &pwr_info->gear_tx); 2350 if (!pwr_info->gear_tx) { 2351 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR), 2352 &pwr_info->gear_tx); 2353 if (!pwr_info->gear_tx) { 2354 dev_err(hba->dev, "%s: invalid max pwm tx gear read = %d\n", 2355 __func__, pwr_info->gear_tx); 2356 return -EINVAL; 2357 } 2358 pwr_info->pwr_tx = SLOWAUTO_MODE; 2359 } 2360 2361 hba->max_pwr_info.is_valid = true; 2362 return 0; 2363} 2364 2365static int ufshcd_change_power_mode(struct ufs_hba *hba, 2366 struct ufs_pa_layer_attr *pwr_mode) 2367{ 2368 int ret; 2369 2370 /* if already configured to the requested pwr_mode */ 2371 if (pwr_mode->gear_rx == hba->pwr_info.gear_rx && 2372 pwr_mode->gear_tx == hba->pwr_info.gear_tx && 2373 pwr_mode->lane_rx == hba->pwr_info.lane_rx && 2374 pwr_mode->lane_tx == hba->pwr_info.lane_tx && 2375 pwr_mode->pwr_rx == hba->pwr_info.pwr_rx && 2376 pwr_mode->pwr_tx == hba->pwr_info.pwr_tx && 2377 pwr_mode->hs_rate == hba->pwr_info.hs_rate) { 2378 dev_dbg(hba->dev, "%s: power already configured\n", __func__); 2379 return 0; 2380 } 2381 2382 /* 2383 * Configure attributes for power mode change with below. 2384 * - PA_RXGEAR, PA_ACTIVERXDATALANES, PA_RXTERMINATION, 2385 * - PA_TXGEAR, PA_ACTIVETXDATALANES, PA_TXTERMINATION, 2386 * - PA_HSSERIES 2387 */ 2388 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXGEAR), pwr_mode->gear_rx); 2389 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVERXDATALANES), 2390 pwr_mode->lane_rx); 2391 if (pwr_mode->pwr_rx == FASTAUTO_MODE || 2392 pwr_mode->pwr_rx == FAST_MODE) 2393 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), TRUE); 2394 else 2395 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), FALSE); 2396 2397 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXGEAR), pwr_mode->gear_tx); 2398 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVETXDATALANES), 2399 pwr_mode->lane_tx); 2400 if (pwr_mode->pwr_tx == FASTAUTO_MODE || 2401 pwr_mode->pwr_tx == FAST_MODE) 2402 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), TRUE); 2403 else 2404 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), FALSE); 2405 2406 if (pwr_mode->pwr_rx == FASTAUTO_MODE || 2407 pwr_mode->pwr_tx == FASTAUTO_MODE || 2408 pwr_mode->pwr_rx == FAST_MODE || 2409 pwr_mode->pwr_tx == FAST_MODE) 2410 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HSSERIES), 2411 pwr_mode->hs_rate); 2412 2413 ret = ufshcd_uic_change_pwr_mode(hba, pwr_mode->pwr_rx << 4 2414 | pwr_mode->pwr_tx); 2415 2416 if (ret) { 2417 dev_err(hba->dev, 2418 "%s: power mode change failed %d\n", __func__, ret); 2419 } else { 2420 if (hba->vops && hba->vops->pwr_change_notify) 2421 hba->vops->pwr_change_notify(hba, 2422 POST_CHANGE, NULL, pwr_mode); 2423 2424 memcpy(&hba->pwr_info, pwr_mode, 2425 sizeof(struct ufs_pa_layer_attr)); 2426 } 2427 2428 return ret; 2429} 2430 2431/** 2432 * ufshcd_config_pwr_mode - configure a new power mode 2433 * @hba: per-adapter instance 2434 * @desired_pwr_mode: desired power configuration 2435 */ 2436static int ufshcd_config_pwr_mode(struct ufs_hba *hba, 2437 struct ufs_pa_layer_attr *desired_pwr_mode) 2438{ 2439 struct ufs_pa_layer_attr final_params = { 0 }; 2440 int ret; 2441 2442 if (hba->vops && hba->vops->pwr_change_notify) 2443 hba->vops->pwr_change_notify(hba, 2444 PRE_CHANGE, desired_pwr_mode, &final_params); 2445 else 2446 memcpy(&final_params, desired_pwr_mode, sizeof(final_params)); 2447 2448 ret = ufshcd_change_power_mode(hba, &final_params); 2449 2450 return ret; 2451} 2452 2453/** 2454 * ufshcd_complete_dev_init() - checks device readiness 2455 * hba: per-adapter instance 2456 * 2457 * Set fDeviceInit flag and poll until device toggles it. 2458 */ 2459static int ufshcd_complete_dev_init(struct ufs_hba *hba) 2460{ 2461 int i, retries, err = 0; 2462 bool flag_res = 1; 2463 2464 for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) { 2465 /* Set the fDeviceInit flag */ 2466 err = ufshcd_query_flag(hba, UPIU_QUERY_OPCODE_SET_FLAG, 2467 QUERY_FLAG_IDN_FDEVICEINIT, NULL); 2468 if (!err || err == -ETIMEDOUT) 2469 break; 2470 dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, err); 2471 } 2472 if (err) { 2473 dev_err(hba->dev, 2474 "%s setting fDeviceInit flag failed with error %d\n", 2475 __func__, err); 2476 goto out; 2477 } 2478 2479 /* poll for max. 100 iterations for fDeviceInit flag to clear */ 2480 for (i = 0; i < 100 && !err && flag_res; i++) { 2481 for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) { 2482 err = ufshcd_query_flag(hba, 2483 UPIU_QUERY_OPCODE_READ_FLAG, 2484 QUERY_FLAG_IDN_FDEVICEINIT, &flag_res); 2485 if (!err || err == -ETIMEDOUT) 2486 break; 2487 dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, 2488 err); 2489 } 2490 } 2491 if (err) 2492 dev_err(hba->dev, 2493 "%s reading fDeviceInit flag failed with error %d\n", 2494 __func__, err); 2495 else if (flag_res) 2496 dev_err(hba->dev, 2497 "%s fDeviceInit was not cleared by the device\n", 2498 __func__); 2499 2500out: 2501 return err; 2502} 2503 2504/** 2505 * ufshcd_make_hba_operational - Make UFS controller operational 2506 * @hba: per adapter instance 2507 * 2508 * To bring UFS host controller to operational state, 2509 * 1. Enable required interrupts 2510 * 2. Configure interrupt aggregation 2511 * 3. Program UTRL and UTMRL base addres 2512 * 4. Configure run-stop-registers 2513 * 2514 * Returns 0 on success, non-zero value on failure 2515 */ 2516static int ufshcd_make_hba_operational(struct ufs_hba *hba) 2517{ 2518 int err = 0; 2519 u32 reg; 2520 2521 /* Enable required interrupts */ 2522 ufshcd_enable_intr(hba, UFSHCD_ENABLE_INTRS); 2523 2524 /* Configure interrupt aggregation */ 2525 ufshcd_config_intr_aggr(hba, hba->nutrs - 1, INT_AGGR_DEF_TO); 2526 2527 /* Configure UTRL and UTMRL base address registers */ 2528 ufshcd_writel(hba, lower_32_bits(hba->utrdl_dma_addr), 2529 REG_UTP_TRANSFER_REQ_LIST_BASE_L); 2530 ufshcd_writel(hba, upper_32_bits(hba->utrdl_dma_addr), 2531 REG_UTP_TRANSFER_REQ_LIST_BASE_H); 2532 ufshcd_writel(hba, lower_32_bits(hba->utmrdl_dma_addr), 2533 REG_UTP_TASK_REQ_LIST_BASE_L); 2534 ufshcd_writel(hba, upper_32_bits(hba->utmrdl_dma_addr), 2535 REG_UTP_TASK_REQ_LIST_BASE_H); 2536 2537 /* 2538 * UCRDY, UTMRLDY and UTRLRDY bits must be 1 2539 * DEI, HEI bits must be 0 2540 */ 2541 reg = ufshcd_readl(hba, REG_CONTROLLER_STATUS); 2542 if (!(ufshcd_get_lists_status(reg))) { 2543 ufshcd_enable_run_stop_reg(hba); 2544 } else { 2545 dev_err(hba->dev, 2546 "Host controller not ready to process requests"); 2547 err = -EIO; 2548 goto out; 2549 } 2550 2551out: 2552 return err; 2553} 2554 2555/** 2556 * ufshcd_hba_enable - initialize the controller 2557 * @hba: per adapter instance 2558 * 2559 * The controller resets itself and controller firmware initialization 2560 * sequence kicks off. When controller is ready it will set 2561 * the Host Controller Enable bit to 1. 2562 * 2563 * Returns 0 on success, non-zero value on failure 2564 */ 2565static int ufshcd_hba_enable(struct ufs_hba *hba) 2566{ 2567 int retry; 2568 2569 /* 2570 * msleep of 1 and 5 used in this function might result in msleep(20), 2571 * but it was necessary to send the UFS FPGA to reset mode during 2572 * development and testing of this driver. msleep can be changed to 2573 * mdelay and retry count can be reduced based on the controller. 2574 */ 2575 if (!ufshcd_is_hba_active(hba)) { 2576 2577 /* change controller state to "reset state" */ 2578 ufshcd_hba_stop(hba); 2579 2580 /* 2581 * This delay is based on the testing done with UFS host 2582 * controller FPGA. The delay can be changed based on the 2583 * host controller used. 2584 */ 2585 msleep(5); 2586 } 2587 2588 /* UniPro link is disabled at this point */ 2589 ufshcd_set_link_off(hba); 2590 2591 if (hba->vops && hba->vops->hce_enable_notify) 2592 hba->vops->hce_enable_notify(hba, PRE_CHANGE); 2593 2594 /* start controller initialization sequence */ 2595 ufshcd_hba_start(hba); 2596 2597 /* 2598 * To initialize a UFS host controller HCE bit must be set to 1. 2599 * During initialization the HCE bit value changes from 1->0->1. 2600 * When the host controller completes initialization sequence 2601 * it sets the value of HCE bit to 1. The same HCE bit is read back 2602 * to check if the controller has completed initialization sequence. 2603 * So without this delay the value HCE = 1, set in the previous 2604 * instruction might be read back. 2605 * This delay can be changed based on the controller. 2606 */ 2607 msleep(1); 2608 2609 /* wait for the host controller to complete initialization */ 2610 retry = 10; 2611 while (ufshcd_is_hba_active(hba)) { 2612 if (retry) { 2613 retry--; 2614 } else { 2615 dev_err(hba->dev, 2616 "Controller enable failed\n"); 2617 return -EIO; 2618 } 2619 msleep(5); 2620 } 2621 2622 /* enable UIC related interrupts */ 2623 ufshcd_enable_intr(hba, UFSHCD_UIC_MASK); 2624 2625 if (hba->vops && hba->vops->hce_enable_notify) 2626 hba->vops->hce_enable_notify(hba, POST_CHANGE); 2627 2628 return 0; 2629} 2630 2631/** 2632 * ufshcd_link_startup - Initialize unipro link startup 2633 * @hba: per adapter instance 2634 * 2635 * Returns 0 for success, non-zero in case of failure 2636 */ 2637static int ufshcd_link_startup(struct ufs_hba *hba) 2638{ 2639 int ret; 2640 int retries = DME_LINKSTARTUP_RETRIES; 2641 2642 do { 2643 if (hba->vops && hba->vops->link_startup_notify) 2644 hba->vops->link_startup_notify(hba, PRE_CHANGE); 2645 2646 ret = ufshcd_dme_link_startup(hba); 2647 2648 /* check if device is detected by inter-connect layer */ 2649 if (!ret && !ufshcd_is_device_present(hba)) { 2650 dev_err(hba->dev, "%s: Device not present\n", __func__); 2651 ret = -ENXIO; 2652 goto out; 2653 } 2654 2655 /* 2656 * DME link lost indication is only received when link is up, 2657 * but we can't be sure if the link is up until link startup 2658 * succeeds. So reset the local Uni-Pro and try again. 2659 */ 2660 if (ret && ufshcd_hba_enable(hba)) 2661 goto out; 2662 } while (ret && retries--); 2663 2664 if (ret) 2665 /* failed to get the link up... retire */ 2666 goto out; 2667 2668 /* Include any host controller configuration via UIC commands */ 2669 if (hba->vops && hba->vops->link_startup_notify) { 2670 ret = hba->vops->link_startup_notify(hba, POST_CHANGE); 2671 if (ret) 2672 goto out; 2673 } 2674 2675 ret = ufshcd_make_hba_operational(hba); 2676out: 2677 if (ret) 2678 dev_err(hba->dev, "link startup failed %d\n", ret); 2679 return ret; 2680} 2681 2682/** 2683 * ufshcd_verify_dev_init() - Verify device initialization 2684 * @hba: per-adapter instance 2685 * 2686 * Send NOP OUT UPIU and wait for NOP IN response to check whether the 2687 * device Transport Protocol (UTP) layer is ready after a reset. 2688 * If the UTP layer at the device side is not initialized, it may 2689 * not respond with NOP IN UPIU within timeout of %NOP_OUT_TIMEOUT 2690 * and we retry sending NOP OUT for %NOP_OUT_RETRIES iterations. 2691 */ 2692static int ufshcd_verify_dev_init(struct ufs_hba *hba) 2693{ 2694 int err = 0; 2695 int retries; 2696 2697 ufshcd_hold(hba, false); 2698 mutex_lock(&hba->dev_cmd.lock); 2699 for (retries = NOP_OUT_RETRIES; retries > 0; retries--) { 2700 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_NOP, 2701 NOP_OUT_TIMEOUT); 2702 2703 if (!err || err == -ETIMEDOUT) 2704 break; 2705 2706 dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, err); 2707 } 2708 mutex_unlock(&hba->dev_cmd.lock); 2709 ufshcd_release(hba); 2710 2711 if (err) 2712 dev_err(hba->dev, "%s: NOP OUT failed %d\n", __func__, err); 2713 return err; 2714} 2715 2716/** 2717 * ufshcd_set_queue_depth - set lun queue depth 2718 * @sdev: pointer to SCSI device 2719 * 2720 * Read bLUQueueDepth value and activate scsi tagged command 2721 * queueing. For WLUN, queue depth is set to 1. For best-effort 2722 * cases (bLUQueueDepth = 0) the queue depth is set to a maximum 2723 * value that host can queue. 2724 */ 2725static void ufshcd_set_queue_depth(struct scsi_device *sdev) 2726{ 2727 int ret = 0; 2728 u8 lun_qdepth; 2729 struct ufs_hba *hba; 2730 2731 hba = shost_priv(sdev->host); 2732 2733 lun_qdepth = hba->nutrs; 2734 ret = ufshcd_read_unit_desc_param(hba, 2735 ufshcd_scsi_to_upiu_lun(sdev->lun), 2736 UNIT_DESC_PARAM_LU_Q_DEPTH, 2737 &lun_qdepth, 2738 sizeof(lun_qdepth)); 2739 2740 /* Some WLUN doesn't support unit descriptor */ 2741 if (ret == -EOPNOTSUPP) 2742 lun_qdepth = 1; 2743 else if (!lun_qdepth) 2744 /* eventually, we can figure out the real queue depth */ 2745 lun_qdepth = hba->nutrs; 2746 else 2747 lun_qdepth = min_t(int, lun_qdepth, hba->nutrs); 2748 2749 dev_dbg(hba->dev, "%s: activate tcq with queue depth %d\n", 2750 __func__, lun_qdepth); 2751 scsi_change_queue_depth(sdev, lun_qdepth); 2752} 2753 2754/* 2755 * ufshcd_get_lu_wp - returns the "b_lu_write_protect" from UNIT DESCRIPTOR 2756 * @hba: per-adapter instance 2757 * @lun: UFS device lun id 2758 * @b_lu_write_protect: pointer to buffer to hold the LU's write protect info 2759 * 2760 * Returns 0 in case of success and b_lu_write_protect status would be returned 2761 * @b_lu_write_protect parameter. 2762 * Returns -ENOTSUPP if reading b_lu_write_protect is not supported. 2763 * Returns -EINVAL in case of invalid parameters passed to this function. 2764 */ 2765static int ufshcd_get_lu_wp(struct ufs_hba *hba, 2766 u8 lun, 2767 u8 *b_lu_write_protect) 2768{ 2769 int ret; 2770 2771 if (!b_lu_write_protect) 2772 ret = -EINVAL; 2773 /* 2774 * According to UFS device spec, RPMB LU can't be write 2775 * protected so skip reading bLUWriteProtect parameter for 2776 * it. For other W-LUs, UNIT DESCRIPTOR is not available. 2777 */ 2778 else if (lun >= UFS_UPIU_MAX_GENERAL_LUN) 2779 ret = -ENOTSUPP; 2780 else 2781 ret = ufshcd_read_unit_desc_param(hba, 2782 lun, 2783 UNIT_DESC_PARAM_LU_WR_PROTECT, 2784 b_lu_write_protect, 2785 sizeof(*b_lu_write_protect)); 2786 return ret; 2787} 2788 2789/** 2790 * ufshcd_get_lu_power_on_wp_status - get LU's power on write protect 2791 * status 2792 * @hba: per-adapter instance 2793 * @sdev: pointer to SCSI device 2794 * 2795 */ 2796static inline void ufshcd_get_lu_power_on_wp_status(struct ufs_hba *hba, 2797 struct scsi_device *sdev) 2798{ 2799 if (hba->dev_info.f_power_on_wp_en && 2800 !hba->dev_info.is_lu_power_on_wp) { 2801 u8 b_lu_write_protect; 2802 2803 if (!ufshcd_get_lu_wp(hba, ufshcd_scsi_to_upiu_lun(sdev->lun), 2804 &b_lu_write_protect) && 2805 (b_lu_write_protect == UFS_LU_POWER_ON_WP)) 2806 hba->dev_info.is_lu_power_on_wp = true; 2807 } 2808} 2809 2810/** 2811 * ufshcd_slave_alloc - handle initial SCSI device configurations 2812 * @sdev: pointer to SCSI device 2813 * 2814 * Returns success 2815 */ 2816static int ufshcd_slave_alloc(struct scsi_device *sdev) 2817{ 2818 struct ufs_hba *hba; 2819 2820 hba = shost_priv(sdev->host); 2821 2822 /* Mode sense(6) is not supported by UFS, so use Mode sense(10) */ 2823 sdev->use_10_for_ms = 1; 2824 2825 /* allow SCSI layer to restart the device in case of errors */ 2826 sdev->allow_restart = 1; 2827 2828 /* REPORT SUPPORTED OPERATION CODES is not supported */ 2829 sdev->no_report_opcodes = 1; 2830 2831 2832 ufshcd_set_queue_depth(sdev); 2833 2834 ufshcd_get_lu_power_on_wp_status(hba, sdev); 2835 2836 return 0; 2837} 2838 2839/** 2840 * ufshcd_change_queue_depth - change queue depth 2841 * @sdev: pointer to SCSI device 2842 * @depth: required depth to set 2843 * 2844 * Change queue depth and make sure the max. limits are not crossed. 2845 */ 2846static int ufshcd_change_queue_depth(struct scsi_device *sdev, int depth) 2847{ 2848 struct ufs_hba *hba = shost_priv(sdev->host); 2849 2850 if (depth > hba->nutrs) 2851 depth = hba->nutrs; 2852 return scsi_change_queue_depth(sdev, depth); 2853} 2854 2855/** 2856 * ufshcd_slave_configure - adjust SCSI device configurations 2857 * @sdev: pointer to SCSI device 2858 */ 2859static int ufshcd_slave_configure(struct scsi_device *sdev) 2860{ 2861 struct request_queue *q = sdev->request_queue; 2862 2863 blk_queue_update_dma_pad(q, PRDT_DATA_BYTE_COUNT_PAD - 1); 2864 blk_queue_max_segment_size(q, PRDT_DATA_BYTE_COUNT_MAX); 2865 2866 return 0; 2867} 2868 2869/** 2870 * ufshcd_slave_destroy - remove SCSI device configurations 2871 * @sdev: pointer to SCSI device 2872 */ 2873static void ufshcd_slave_destroy(struct scsi_device *sdev) 2874{ 2875 struct ufs_hba *hba; 2876 2877 hba = shost_priv(sdev->host); 2878 /* Drop the reference as it won't be needed anymore */ 2879 if (ufshcd_scsi_to_upiu_lun(sdev->lun) == UFS_UPIU_UFS_DEVICE_WLUN) { 2880 unsigned long flags; 2881 2882 spin_lock_irqsave(hba->host->host_lock, flags); 2883 hba->sdev_ufs_device = NULL; 2884 spin_unlock_irqrestore(hba->host->host_lock, flags); 2885 } 2886} 2887 2888/** 2889 * ufshcd_task_req_compl - handle task management request completion 2890 * @hba: per adapter instance 2891 * @index: index of the completed request 2892 * @resp: task management service response 2893 * 2894 * Returns non-zero value on error, zero on success 2895 */ 2896static int ufshcd_task_req_compl(struct ufs_hba *hba, u32 index, u8 *resp) 2897{ 2898 struct utp_task_req_desc *task_req_descp; 2899 struct utp_upiu_task_rsp *task_rsp_upiup; 2900 unsigned long flags; 2901 int ocs_value; 2902 int task_result; 2903 2904 spin_lock_irqsave(hba->host->host_lock, flags); 2905 2906 /* Clear completed tasks from outstanding_tasks */ 2907 __clear_bit(index, &hba->outstanding_tasks); 2908 2909 task_req_descp = hba->utmrdl_base_addr; 2910 ocs_value = ufshcd_get_tmr_ocs(&task_req_descp[index]); 2911 2912 if (ocs_value == OCS_SUCCESS) { 2913 task_rsp_upiup = (struct utp_upiu_task_rsp *) 2914 task_req_descp[index].task_rsp_upiu; 2915 task_result = be32_to_cpu(task_rsp_upiup->header.dword_1); 2916 task_result = ((task_result & MASK_TASK_RESPONSE) >> 8); 2917 if (resp) 2918 *resp = (u8)task_result; 2919 } else { 2920 dev_err(hba->dev, "%s: failed, ocs = 0x%x\n", 2921 __func__, ocs_value); 2922 } 2923 spin_unlock_irqrestore(hba->host->host_lock, flags); 2924 2925 return ocs_value; 2926} 2927 2928/** 2929 * ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status 2930 * @lrb: pointer to local reference block of completed command 2931 * @scsi_status: SCSI command status 2932 * 2933 * Returns value base on SCSI command status 2934 */ 2935static inline int 2936ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status) 2937{ 2938 int result = 0; 2939 2940 switch (scsi_status) { 2941 case SAM_STAT_CHECK_CONDITION: 2942 ufshcd_copy_sense_data(lrbp); 2943 case SAM_STAT_GOOD: 2944 result |= DID_OK << 16 | 2945 COMMAND_COMPLETE << 8 | 2946 scsi_status; 2947 break; 2948 case SAM_STAT_TASK_SET_FULL: 2949 case SAM_STAT_BUSY: 2950 case SAM_STAT_TASK_ABORTED: 2951 ufshcd_copy_sense_data(lrbp); 2952 result |= scsi_status; 2953 break; 2954 default: 2955 result |= DID_ERROR << 16; 2956 break; 2957 } /* end of switch */ 2958 2959 return result; 2960} 2961 2962/** 2963 * ufshcd_transfer_rsp_status - Get overall status of the response 2964 * @hba: per adapter instance 2965 * @lrb: pointer to local reference block of completed command 2966 * 2967 * Returns result of the command to notify SCSI midlayer 2968 */ 2969static inline int 2970ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) 2971{ 2972 int result = 0; 2973 int scsi_status; 2974 int ocs; 2975 2976 /* overall command status of utrd */ 2977 ocs = ufshcd_get_tr_ocs(lrbp); 2978 2979 switch (ocs) { 2980 case OCS_SUCCESS: 2981 result = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr); 2982 2983 switch (result) { 2984 case UPIU_TRANSACTION_RESPONSE: 2985 /* 2986 * get the response UPIU result to extract 2987 * the SCSI command status 2988 */ 2989 result = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr); 2990 2991 /* 2992 * get the result based on SCSI status response 2993 * to notify the SCSI midlayer of the command status 2994 */ 2995 scsi_status = result & MASK_SCSI_STATUS; 2996 result = ufshcd_scsi_cmd_status(lrbp, scsi_status); 2997 2998 if (ufshcd_is_exception_event(lrbp->ucd_rsp_ptr)) 2999 schedule_work(&hba->eeh_work); 3000 break; 3001 case UPIU_TRANSACTION_REJECT_UPIU: 3002 /* TODO: handle Reject UPIU Response */ 3003 result = DID_ERROR << 16; 3004 dev_err(hba->dev, 3005 "Reject UPIU not fully implemented\n"); 3006 break; 3007 default: 3008 result = DID_ERROR << 16; 3009 dev_err(hba->dev, 3010 "Unexpected request response code = %x\n", 3011 result); 3012 break; 3013 } 3014 break; 3015 case OCS_ABORTED: 3016 result |= DID_ABORT << 16; 3017 break; 3018 case OCS_INVALID_COMMAND_STATUS: 3019 result |= DID_REQUEUE << 16; 3020 break; 3021 case OCS_INVALID_CMD_TABLE_ATTR: 3022 case OCS_INVALID_PRDT_ATTR: 3023 case OCS_MISMATCH_DATA_BUF_SIZE: 3024 case OCS_MISMATCH_RESP_UPIU_SIZE: 3025 case OCS_PEER_COMM_FAILURE: 3026 case OCS_FATAL_ERROR: 3027 default: 3028 result |= DID_ERROR << 16; 3029 dev_err(hba->dev, 3030 "OCS error from controller = %x\n", ocs); 3031 break; 3032 } /* end of switch */ 3033 3034 return result; 3035} 3036 3037/** 3038 * ufshcd_uic_cmd_compl - handle completion of uic command 3039 * @hba: per adapter instance 3040 * @intr_status: interrupt status generated by the controller 3041 */ 3042static void ufshcd_uic_cmd_compl(struct ufs_hba *hba, u32 intr_status) 3043{ 3044 if ((intr_status & UIC_COMMAND_COMPL) && hba->active_uic_cmd) { 3045 hba->active_uic_cmd->argument2 |= 3046 ufshcd_get_uic_cmd_result(hba); 3047 hba->active_uic_cmd->argument3 = 3048 ufshcd_get_dme_attr_val(hba); 3049 complete(&hba->active_uic_cmd->done); 3050 } 3051 3052 if ((intr_status & UFSHCD_UIC_PWR_MASK) && hba->uic_async_done) 3053 complete(hba->uic_async_done); 3054} 3055 3056/** 3057 * ufshcd_transfer_req_compl - handle SCSI and query command completion 3058 * @hba: per adapter instance 3059 */ 3060static void ufshcd_transfer_req_compl(struct ufs_hba *hba) 3061{ 3062 struct ufshcd_lrb *lrbp; 3063 struct scsi_cmnd *cmd; 3064 unsigned long completed_reqs; 3065 u32 tr_doorbell; 3066 int result; 3067 int index; 3068 3069 /* Resetting interrupt aggregation counters first and reading the 3070 * DOOR_BELL afterward allows us to handle all the completed requests. 3071 * In order to prevent other interrupts starvation the DB is read once 3072 * after reset. The down side of this solution is the possibility of 3073 * false interrupt if device completes another request after resetting 3074 * aggregation and before reading the DB. 3075 */ 3076 ufshcd_reset_intr_aggr(hba); 3077 3078 tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); 3079 completed_reqs = tr_doorbell ^ hba->outstanding_reqs; 3080 3081 for_each_set_bit(index, &completed_reqs, hba->nutrs) { 3082 lrbp = &hba->lrb[index]; 3083 cmd = lrbp->cmd; 3084 if (cmd) { 3085 result = ufshcd_transfer_rsp_status(hba, lrbp); 3086 scsi_dma_unmap(cmd); 3087 cmd->result = result; 3088 /* Mark completed command as NULL in LRB */ 3089 lrbp->cmd = NULL; 3090 clear_bit_unlock(index, &hba->lrb_in_use); 3091 /* Do not touch lrbp after scsi done */ 3092 cmd->scsi_done(cmd); 3093 __ufshcd_release(hba); 3094 } else if (lrbp->command_type == UTP_CMD_TYPE_DEV_MANAGE) { 3095 if (hba->dev_cmd.complete) 3096 complete(hba->dev_cmd.complete); 3097 } 3098 } 3099 3100 /* clear corresponding bits of completed commands */ 3101 hba->outstanding_reqs ^= completed_reqs; 3102 3103 ufshcd_clk_scaling_update_busy(hba); 3104 3105 /* we might have free'd some tags above */ 3106 wake_up(&hba->dev_cmd.tag_wq); 3107} 3108 3109/** 3110 * ufshcd_disable_ee - disable exception event 3111 * @hba: per-adapter instance 3112 * @mask: exception event to disable 3113 * 3114 * Disables exception event in the device so that the EVENT_ALERT 3115 * bit is not set. 3116 * 3117 * Returns zero on success, non-zero error value on failure. 3118 */ 3119static int ufshcd_disable_ee(struct ufs_hba *hba, u16 mask) 3120{ 3121 int err = 0; 3122 u32 val; 3123 3124 if (!(hba->ee_ctrl_mask & mask)) 3125 goto out; 3126 3127 val = hba->ee_ctrl_mask & ~mask; 3128 val &= 0xFFFF; /* 2 bytes */ 3129 err = ufshcd_query_attr(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, 3130 QUERY_ATTR_IDN_EE_CONTROL, 0, 0, &val); 3131 if (!err) 3132 hba->ee_ctrl_mask &= ~mask; 3133out: 3134 return err; 3135} 3136 3137/** 3138 * ufshcd_enable_ee - enable exception event 3139 * @hba: per-adapter instance 3140 * @mask: exception event to enable 3141 * 3142 * Enable corresponding exception event in the device to allow 3143 * device to alert host in critical scenarios. 3144 * 3145 * Returns zero on success, non-zero error value on failure. 3146 */ 3147static int ufshcd_enable_ee(struct ufs_hba *hba, u16 mask) 3148{ 3149 int err = 0; 3150 u32 val; 3151 3152 if (hba->ee_ctrl_mask & mask) 3153 goto out; 3154 3155 val = hba->ee_ctrl_mask | mask; 3156 val &= 0xFFFF; /* 2 bytes */ 3157 err = ufshcd_query_attr(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, 3158 QUERY_ATTR_IDN_EE_CONTROL, 0, 0, &val); 3159 if (!err) 3160 hba->ee_ctrl_mask |= mask; 3161out: 3162 return err; 3163} 3164 3165/** 3166 * ufshcd_enable_auto_bkops - Allow device managed BKOPS 3167 * @hba: per-adapter instance 3168 * 3169 * Allow device to manage background operations on its own. Enabling 3170 * this might lead to inconsistent latencies during normal data transfers 3171 * as the device is allowed to manage its own way of handling background 3172 * operations. 3173 * 3174 * Returns zero on success, non-zero on failure. 3175 */ 3176static int ufshcd_enable_auto_bkops(struct ufs_hba *hba) 3177{ 3178 int err = 0; 3179 3180 if (hba->auto_bkops_enabled) 3181 goto out; 3182 3183 err = ufshcd_query_flag(hba, UPIU_QUERY_OPCODE_SET_FLAG, 3184 QUERY_FLAG_IDN_BKOPS_EN, NULL); 3185 if (err) { 3186 dev_err(hba->dev, "%s: failed to enable bkops %d\n", 3187 __func__, err); 3188 goto out; 3189 } 3190 3191 hba->auto_bkops_enabled = true; 3192 3193 /* No need of URGENT_BKOPS exception from the device */ 3194 err = ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS); 3195 if (err) 3196 dev_err(hba->dev, "%s: failed to disable exception event %d\n", 3197 __func__, err); 3198out: 3199 return err; 3200} 3201 3202/** 3203 * ufshcd_disable_auto_bkops - block device in doing background operations 3204 * @hba: per-adapter instance 3205 * 3206 * Disabling background operations improves command response latency but 3207 * has drawback of device moving into critical state where the device is 3208 * not-operable. Make sure to call ufshcd_enable_auto_bkops() whenever the 3209 * host is idle so that BKOPS are managed effectively without any negative 3210 * impacts. 3211 * 3212 * Returns zero on success, non-zero on failure. 3213 */ 3214static int ufshcd_disable_auto_bkops(struct ufs_hba *hba) 3215{ 3216 int err = 0; 3217 3218 if (!hba->auto_bkops_enabled) 3219 goto out; 3220 3221 /* 3222 * If host assisted BKOPs is to be enabled, make sure 3223 * urgent bkops exception is allowed. 3224 */ 3225 err = ufshcd_enable_ee(hba, MASK_EE_URGENT_BKOPS); 3226 if (err) { 3227 dev_err(hba->dev, "%s: failed to enable exception event %d\n", 3228 __func__, err); 3229 goto out; 3230 } 3231 3232 err = ufshcd_query_flag(hba, UPIU_QUERY_OPCODE_CLEAR_FLAG, 3233 QUERY_FLAG_IDN_BKOPS_EN, NULL); 3234 if (err) { 3235 dev_err(hba->dev, "%s: failed to disable bkops %d\n", 3236 __func__, err); 3237 ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS); 3238 goto out; 3239 } 3240 3241 hba->auto_bkops_enabled = false; 3242out: 3243 return err; 3244} 3245 3246/** 3247 * ufshcd_force_reset_auto_bkops - force enable of auto bkops 3248 * @hba: per adapter instance 3249 * 3250 * After a device reset the device may toggle the BKOPS_EN flag 3251 * to default value. The s/w tracking variables should be updated 3252 * as well. Do this by forcing enable of auto bkops. 3253 */ 3254static void ufshcd_force_reset_auto_bkops(struct ufs_hba *hba) 3255{ 3256 hba->auto_bkops_enabled = false; 3257 hba->ee_ctrl_mask |= MASK_EE_URGENT_BKOPS; 3258 ufshcd_enable_auto_bkops(hba); 3259} 3260 3261static inline int ufshcd_get_bkops_status(struct ufs_hba *hba, u32 *status) 3262{ 3263 return ufshcd_query_attr(hba, UPIU_QUERY_OPCODE_READ_ATTR, 3264 QUERY_ATTR_IDN_BKOPS_STATUS, 0, 0, status); 3265} 3266 3267/** 3268 * ufshcd_bkops_ctrl - control the auto bkops based on current bkops status 3269 * @hba: per-adapter instance 3270 * @status: bkops_status value 3271 * 3272 * Read the bkops_status from the UFS device and Enable fBackgroundOpsEn 3273 * flag in the device to permit background operations if the device 3274 * bkops_status is greater than or equal to "status" argument passed to 3275 * this function, disable otherwise. 3276 * 3277 * Returns 0 for success, non-zero in case of failure. 3278 * 3279 * NOTE: Caller of this function can check the "hba->auto_bkops_enabled" flag 3280 * to know whether auto bkops is enabled or disabled after this function 3281 * returns control to it. 3282 */ 3283static int ufshcd_bkops_ctrl(struct ufs_hba *hba, 3284 enum bkops_status status) 3285{ 3286 int err; 3287 u32 curr_status = 0; 3288 3289 err = ufshcd_get_bkops_status(hba, &curr_status); 3290 if (err) { 3291 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n", 3292 __func__, err); 3293 goto out; 3294 } else if (curr_status > BKOPS_STATUS_MAX) { 3295 dev_err(hba->dev, "%s: invalid BKOPS status %d\n", 3296 __func__, curr_status); 3297 err = -EINVAL; 3298 goto out; 3299 } 3300 3301 if (curr_status >= status) 3302 err = ufshcd_enable_auto_bkops(hba); 3303 else 3304 err = ufshcd_disable_auto_bkops(hba); 3305out: 3306 return err; 3307} 3308 3309/** 3310 * ufshcd_urgent_bkops - handle urgent bkops exception event 3311 * @hba: per-adapter instance 3312 * 3313 * Enable fBackgroundOpsEn flag in the device to permit background 3314 * operations. 3315 * 3316 * If BKOPs is enabled, this function returns 0, 1 if the bkops in not enabled 3317 * and negative error value for any other failure. 3318 */ 3319static int ufshcd_urgent_bkops(struct ufs_hba *hba) 3320{ 3321 return ufshcd_bkops_ctrl(hba, BKOPS_STATUS_PERF_IMPACT); 3322} 3323 3324static inline int ufshcd_get_ee_status(struct ufs_hba *hba, u32 *status) 3325{ 3326 return ufshcd_query_attr(hba, UPIU_QUERY_OPCODE_READ_ATTR, 3327 QUERY_ATTR_IDN_EE_STATUS, 0, 0, status); 3328} 3329 3330/** 3331 * ufshcd_exception_event_handler - handle exceptions raised by device 3332 * @work: pointer to work data 3333 * 3334 * Read bExceptionEventStatus attribute from the device and handle the 3335 * exception event accordingly. 3336 */ 3337static void ufshcd_exception_event_handler(struct work_struct *work) 3338{ 3339 struct ufs_hba *hba; 3340 int err; 3341 u32 status = 0; 3342 hba = container_of(work, struct ufs_hba, eeh_work); 3343 3344 pm_runtime_get_sync(hba->dev); 3345 err = ufshcd_get_ee_status(hba, &status); 3346 if (err) { 3347 dev_err(hba->dev, "%s: failed to get exception status %d\n", 3348 __func__, err); 3349 goto out; 3350 } 3351 3352 status &= hba->ee_ctrl_mask; 3353 if (status & MASK_EE_URGENT_BKOPS) { 3354 err = ufshcd_urgent_bkops(hba); 3355 if (err < 0) 3356 dev_err(hba->dev, "%s: failed to handle urgent bkops %d\n", 3357 __func__, err); 3358 } 3359out: 3360 pm_runtime_put_sync(hba->dev); 3361 return; 3362} 3363 3364/** 3365 * ufshcd_err_handler - handle UFS errors that require s/w attention 3366 * @work: pointer to work structure 3367 */ 3368static void ufshcd_err_handler(struct work_struct *work) 3369{ 3370 struct ufs_hba *hba; 3371 unsigned long flags; 3372 u32 err_xfer = 0; 3373 u32 err_tm = 0; 3374 int err = 0; 3375 int tag; 3376 3377 hba = container_of(work, struct ufs_hba, eh_work); 3378 3379 pm_runtime_get_sync(hba->dev); 3380 ufshcd_hold(hba, false); 3381 3382 spin_lock_irqsave(hba->host->host_lock, flags); 3383 if (hba->ufshcd_state == UFSHCD_STATE_RESET) { 3384 spin_unlock_irqrestore(hba->host->host_lock, flags); 3385 goto out; 3386 } 3387 3388 hba->ufshcd_state = UFSHCD_STATE_RESET; 3389 ufshcd_set_eh_in_progress(hba); 3390 3391 /* Complete requests that have door-bell cleared by h/w */ 3392 ufshcd_transfer_req_compl(hba); 3393 ufshcd_tmc_handler(hba); 3394 spin_unlock_irqrestore(hba->host->host_lock, flags); 3395 3396 /* Clear pending transfer requests */ 3397 for_each_set_bit(tag, &hba->outstanding_reqs, hba->nutrs) 3398 if (ufshcd_clear_cmd(hba, tag)) 3399 err_xfer |= 1 << tag; 3400 3401 /* Clear pending task management requests */ 3402 for_each_set_bit(tag, &hba->outstanding_tasks, hba->nutmrs) 3403 if (ufshcd_clear_tm_cmd(hba, tag)) 3404 err_tm |= 1 << tag; 3405 3406 /* Complete the requests that are cleared by s/w */ 3407 spin_lock_irqsave(hba->host->host_lock, flags); 3408 ufshcd_transfer_req_compl(hba); 3409 ufshcd_tmc_handler(hba); 3410 spin_unlock_irqrestore(hba->host->host_lock, flags); 3411 3412 /* Fatal errors need reset */ 3413 if (err_xfer || err_tm || (hba->saved_err & INT_FATAL_ERRORS) || 3414 ((hba->saved_err & UIC_ERROR) && 3415 (hba->saved_uic_err & UFSHCD_UIC_DL_PA_INIT_ERROR))) { 3416 err = ufshcd_reset_and_restore(hba); 3417 if (err) { 3418 dev_err(hba->dev, "%s: reset and restore failed\n", 3419 __func__); 3420 hba->ufshcd_state = UFSHCD_STATE_ERROR; 3421 } 3422 /* 3423 * Inform scsi mid-layer that we did reset and allow to handle 3424 * Unit Attention properly. 3425 */ 3426 scsi_report_bus_reset(hba->host, 0); 3427 hba->saved_err = 0; 3428 hba->saved_uic_err = 0; 3429 } 3430 ufshcd_clear_eh_in_progress(hba); 3431 3432out: 3433 scsi_unblock_requests(hba->host); 3434 ufshcd_release(hba); 3435 pm_runtime_put_sync(hba->dev); 3436} 3437 3438/** 3439 * ufshcd_update_uic_error - check and set fatal UIC error flags. 3440 * @hba: per-adapter instance 3441 */ 3442static void ufshcd_update_uic_error(struct ufs_hba *hba) 3443{ 3444 u32 reg; 3445 3446 /* PA_INIT_ERROR is fatal and needs UIC reset */ 3447 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DATA_LINK_LAYER); 3448 if (reg & UIC_DATA_LINK_LAYER_ERROR_PA_INIT) 3449 hba->uic_error |= UFSHCD_UIC_DL_PA_INIT_ERROR; 3450 3451 /* UIC NL/TL/DME errors needs software retry */ 3452 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_NETWORK_LAYER); 3453 if (reg) 3454 hba->uic_error |= UFSHCD_UIC_NL_ERROR; 3455 3456 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_TRANSPORT_LAYER); 3457 if (reg) 3458 hba->uic_error |= UFSHCD_UIC_TL_ERROR; 3459 3460 reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DME); 3461 if (reg) 3462 hba->uic_error |= UFSHCD_UIC_DME_ERROR; 3463 3464 dev_dbg(hba->dev, "%s: UIC error flags = 0x%08x\n", 3465 __func__, hba->uic_error); 3466} 3467 3468/** 3469 * ufshcd_check_errors - Check for errors that need s/w attention 3470 * @hba: per-adapter instance 3471 */ 3472static void ufshcd_check_errors(struct ufs_hba *hba) 3473{ 3474 bool queue_eh_work = false; 3475 3476 if (hba->errors & INT_FATAL_ERRORS) 3477 queue_eh_work = true; 3478 3479 if (hba->errors & UIC_ERROR) { 3480 hba->uic_error = 0; 3481 ufshcd_update_uic_error(hba); 3482 if (hba->uic_error) 3483 queue_eh_work = true; 3484 } 3485 3486 if (queue_eh_work) { 3487 /* handle fatal errors only when link is functional */ 3488 if (hba->ufshcd_state == UFSHCD_STATE_OPERATIONAL) { 3489 /* block commands from scsi mid-layer */ 3490 scsi_block_requests(hba->host); 3491 3492 /* transfer error masks to sticky bits */ 3493 hba->saved_err |= hba->errors; 3494 hba->saved_uic_err |= hba->uic_error; 3495 3496 hba->ufshcd_state = UFSHCD_STATE_ERROR; 3497 schedule_work(&hba->eh_work); 3498 } 3499 } 3500 /* 3501 * if (!queue_eh_work) - 3502 * Other errors are either non-fatal where host recovers 3503 * itself without s/w intervention or errors that will be 3504 * handled by the SCSI core layer. 3505 */ 3506} 3507 3508/** 3509 * ufshcd_tmc_handler - handle task management function completion 3510 * @hba: per adapter instance 3511 */ 3512static void ufshcd_tmc_handler(struct ufs_hba *hba) 3513{ 3514 u32 tm_doorbell; 3515 3516 tm_doorbell = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL); 3517 hba->tm_condition = tm_doorbell ^ hba->outstanding_tasks; 3518 wake_up(&hba->tm_wq); 3519} 3520 3521/** 3522 * ufshcd_sl_intr - Interrupt service routine 3523 * @hba: per adapter instance 3524 * @intr_status: contains interrupts generated by the controller 3525 */ 3526static void ufshcd_sl_intr(struct ufs_hba *hba, u32 intr_status) 3527{ 3528 hba->errors = UFSHCD_ERROR_MASK & intr_status; 3529 if (hba->errors) 3530 ufshcd_check_errors(hba); 3531 3532 if (intr_status & UFSHCD_UIC_MASK) 3533 ufshcd_uic_cmd_compl(hba, intr_status); 3534 3535 if (intr_status & UTP_TASK_REQ_COMPL) 3536 ufshcd_tmc_handler(hba); 3537 3538 if (intr_status & UTP_TRANSFER_REQ_COMPL) 3539 ufshcd_transfer_req_compl(hba); 3540} 3541 3542/** 3543 * ufshcd_intr - Main interrupt service routine 3544 * @irq: irq number 3545 * @__hba: pointer to adapter instance 3546 * 3547 * Returns IRQ_HANDLED - If interrupt is valid 3548 * IRQ_NONE - If invalid interrupt 3549 */ 3550static irqreturn_t ufshcd_intr(int irq, void *__hba) 3551{ 3552 u32 intr_status; 3553 irqreturn_t retval = IRQ_NONE; 3554 struct ufs_hba *hba = __hba; 3555 3556 spin_lock(hba->host->host_lock); 3557 intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS); 3558 3559 if (intr_status) { 3560 ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS); 3561 ufshcd_sl_intr(hba, intr_status); 3562 retval = IRQ_HANDLED; 3563 } 3564 spin_unlock(hba->host->host_lock); 3565 return retval; 3566} 3567 3568static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag) 3569{ 3570 int err = 0; 3571 u32 mask = 1 << tag; 3572 unsigned long flags; 3573 3574 if (!test_bit(tag, &hba->outstanding_tasks)) 3575 goto out; 3576 3577 spin_lock_irqsave(hba->host->host_lock, flags); 3578 ufshcd_writel(hba, ~(1 << tag), REG_UTP_TASK_REQ_LIST_CLEAR); 3579 spin_unlock_irqrestore(hba->host->host_lock, flags); 3580 3581 /* poll for max. 1 sec to clear door bell register by h/w */ 3582 err = ufshcd_wait_for_register(hba, 3583 REG_UTP_TASK_REQ_DOOR_BELL, 3584 mask, 0, 1000, 1000); 3585out: 3586 return err; 3587} 3588 3589/** 3590 * ufshcd_issue_tm_cmd - issues task management commands to controller 3591 * @hba: per adapter instance 3592 * @lun_id: LUN ID to which TM command is sent 3593 * @task_id: task ID to which the TM command is applicable 3594 * @tm_function: task management function opcode 3595 * @tm_response: task management service response return value 3596 * 3597 * Returns non-zero value on error, zero on success. 3598 */ 3599static int ufshcd_issue_tm_cmd(struct ufs_hba *hba, int lun_id, int task_id, 3600 u8 tm_function, u8 *tm_response) 3601{ 3602 struct utp_task_req_desc *task_req_descp; 3603 struct utp_upiu_task_req *task_req_upiup; 3604 struct Scsi_Host *host; 3605 unsigned long flags; 3606 int free_slot; 3607 int err; 3608 int task_tag; 3609 3610 host = hba->host; 3611 3612 /* 3613 * Get free slot, sleep if slots are unavailable. 3614 * Even though we use wait_event() which sleeps indefinitely, 3615 * the maximum wait time is bounded by %TM_CMD_TIMEOUT. 3616 */ 3617 wait_event(hba->tm_tag_wq, ufshcd_get_tm_free_slot(hba, &free_slot)); 3618 ufshcd_hold(hba, false); 3619 3620 spin_lock_irqsave(host->host_lock, flags); 3621 task_req_descp = hba->utmrdl_base_addr; 3622 task_req_descp += free_slot; 3623 3624 /* Configure task request descriptor */ 3625 task_req_descp->header.dword_0 = cpu_to_le32(UTP_REQ_DESC_INT_CMD); 3626 task_req_descp->header.dword_2 = 3627 cpu_to_le32(OCS_INVALID_COMMAND_STATUS); 3628 3629 /* Configure task request UPIU */ 3630 task_req_upiup = 3631 (struct utp_upiu_task_req *) task_req_descp->task_req_upiu; 3632 task_tag = hba->nutrs + free_slot; 3633 task_req_upiup->header.dword_0 = 3634 UPIU_HEADER_DWORD(UPIU_TRANSACTION_TASK_REQ, 0, 3635 lun_id, task_tag); 3636 task_req_upiup->header.dword_1 = 3637 UPIU_HEADER_DWORD(0, tm_function, 0, 0); 3638 /* 3639 * The host shall provide the same value for LUN field in the basic 3640 * header and for Input Parameter. 3641 */ 3642 task_req_upiup->input_param1 = cpu_to_be32(lun_id); 3643 task_req_upiup->input_param2 = cpu_to_be32(task_id); 3644 3645 /* send command to the controller */ 3646 __set_bit(free_slot, &hba->outstanding_tasks); 3647 ufshcd_writel(hba, 1 << free_slot, REG_UTP_TASK_REQ_DOOR_BELL); 3648 3649 spin_unlock_irqrestore(host->host_lock, flags); 3650 3651 /* wait until the task management command is completed */ 3652 err = wait_event_timeout(hba->tm_wq, 3653 test_bit(free_slot, &hba->tm_condition), 3654 msecs_to_jiffies(TM_CMD_TIMEOUT)); 3655 if (!err) { 3656 dev_err(hba->dev, "%s: task management cmd 0x%.2x timed-out\n", 3657 __func__, tm_function); 3658 if (ufshcd_clear_tm_cmd(hba, free_slot)) 3659 dev_WARN(hba->dev, "%s: unable clear tm cmd (slot %d) after timeout\n", 3660 __func__, free_slot); 3661 err = -ETIMEDOUT; 3662 } else { 3663 err = ufshcd_task_req_compl(hba, free_slot, tm_response); 3664 } 3665 3666 clear_bit(free_slot, &hba->tm_condition); 3667 ufshcd_put_tm_slot(hba, free_slot); 3668 wake_up(&hba->tm_tag_wq); 3669 3670 ufshcd_release(hba); 3671 return err; 3672} 3673 3674/** 3675 * ufshcd_eh_device_reset_handler - device reset handler registered to 3676 * scsi layer. 3677 * @cmd: SCSI command pointer 3678 * 3679 * Returns SUCCESS/FAILED 3680 */ 3681static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd) 3682{ 3683 struct Scsi_Host *host; 3684 struct ufs_hba *hba; 3685 unsigned int tag; 3686 u32 pos; 3687 int err; 3688 u8 resp = 0xF; 3689 struct ufshcd_lrb *lrbp; 3690 unsigned long flags; 3691 3692 host = cmd->device->host; 3693 hba = shost_priv(host); 3694 tag = cmd->request->tag; 3695 3696 lrbp = &hba->lrb[tag]; 3697 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, 0, UFS_LOGICAL_RESET, &resp); 3698 if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) { 3699 if (!err) 3700 err = resp; 3701 goto out; 3702 } 3703 3704 /* clear the commands that were pending for corresponding LUN */ 3705 for_each_set_bit(pos, &hba->outstanding_reqs, hba->nutrs) { 3706 if (hba->lrb[pos].lun == lrbp->lun) { 3707 err = ufshcd_clear_cmd(hba, pos); 3708 if (err) 3709 break; 3710 } 3711 } 3712 spin_lock_irqsave(host->host_lock, flags); 3713 ufshcd_transfer_req_compl(hba); 3714 spin_unlock_irqrestore(host->host_lock, flags); 3715out: 3716 if (!err) { 3717 err = SUCCESS; 3718 } else { 3719 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err); 3720 err = FAILED; 3721 } 3722 return err; 3723} 3724 3725/** 3726 * ufshcd_abort - abort a specific command 3727 * @cmd: SCSI command pointer 3728 * 3729 * Abort the pending command in device by sending UFS_ABORT_TASK task management 3730 * command, and in host controller by clearing the door-bell register. There can 3731 * be race between controller sending the command to the device while abort is 3732 * issued. To avoid that, first issue UFS_QUERY_TASK to check if the command is 3733 * really issued and then try to abort it. 3734 * 3735 * Returns SUCCESS/FAILED 3736 */ 3737static int ufshcd_abort(struct scsi_cmnd *cmd) 3738{ 3739 struct Scsi_Host *host; 3740 struct ufs_hba *hba; 3741 unsigned long flags; 3742 unsigned int tag; 3743 int err = 0; 3744 int poll_cnt; 3745 u8 resp = 0xF; 3746 struct ufshcd_lrb *lrbp; 3747 u32 reg; 3748 3749 host = cmd->device->host; 3750 hba = shost_priv(host); 3751 tag = cmd->request->tag; 3752 3753 ufshcd_hold(hba, false); 3754 /* If command is already aborted/completed, return SUCCESS */ 3755 if (!(test_bit(tag, &hba->outstanding_reqs))) 3756 goto out; 3757 3758 reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); 3759 if (!(reg & (1 << tag))) { 3760 dev_err(hba->dev, 3761 "%s: cmd was completed, but without a notifying intr, tag = %d", 3762 __func__, tag); 3763 } 3764 3765 lrbp = &hba->lrb[tag]; 3766 for (poll_cnt = 100; poll_cnt; poll_cnt--) { 3767 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag, 3768 UFS_QUERY_TASK, &resp); 3769 if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_SUCCEEDED) { 3770 /* cmd pending in the device */ 3771 break; 3772 } else if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_COMPL) { 3773 /* 3774 * cmd not pending in the device, check if it is 3775 * in transition. 3776 */ 3777 reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); 3778 if (reg & (1 << tag)) { 3779 /* sleep for max. 200us to stabilize */ 3780 usleep_range(100, 200); 3781 continue; 3782 } 3783 /* command completed already */ 3784 goto out; 3785 } else { 3786 if (!err) 3787 err = resp; /* service response error */ 3788 goto out; 3789 } 3790 } 3791 3792 if (!poll_cnt) { 3793 err = -EBUSY; 3794 goto out; 3795 } 3796 3797 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag, 3798 UFS_ABORT_TASK, &resp); 3799 if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) { 3800 if (!err) 3801 err = resp; /* service response error */ 3802 goto out; 3803 } 3804 3805 err = ufshcd_clear_cmd(hba, tag); 3806 if (err) 3807 goto out; 3808 3809 scsi_dma_unmap(cmd); 3810 3811 spin_lock_irqsave(host->host_lock, flags); 3812 __clear_bit(tag, &hba->outstanding_reqs); 3813 hba->lrb[tag].cmd = NULL; 3814 spin_unlock_irqrestore(host->host_lock, flags); 3815 3816 clear_bit_unlock(tag, &hba->lrb_in_use); 3817 wake_up(&hba->dev_cmd.tag_wq); 3818 3819out: 3820 if (!err) { 3821 err = SUCCESS; 3822 } else { 3823 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err); 3824 err = FAILED; 3825 } 3826 3827 /* 3828 * This ufshcd_release() corresponds to the original scsi cmd that got 3829 * aborted here (as we won't get any IRQ for it). 3830 */ 3831 ufshcd_release(hba); 3832 return err; 3833} 3834 3835/** 3836 * ufshcd_host_reset_and_restore - reset and restore host controller 3837 * @hba: per-adapter instance 3838 * 3839 * Note that host controller reset may issue DME_RESET to 3840 * local and remote (device) Uni-Pro stack and the attributes 3841 * are reset to default state. 3842 * 3843 * Returns zero on success, non-zero on failure 3844 */ 3845static int ufshcd_host_reset_and_restore(struct ufs_hba *hba) 3846{ 3847 int err; 3848 unsigned long flags; 3849 3850 /* Reset the host controller */ 3851 spin_lock_irqsave(hba->host->host_lock, flags); 3852 ufshcd_hba_stop(hba); 3853 spin_unlock_irqrestore(hba->host->host_lock, flags); 3854 3855 err = ufshcd_hba_enable(hba); 3856 if (err) 3857 goto out; 3858 3859 /* Establish the link again and restore the device */ 3860 err = ufshcd_probe_hba(hba); 3861 3862 if (!err && (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL)) 3863 err = -EIO; 3864out: 3865 if (err) 3866 dev_err(hba->dev, "%s: Host init failed %d\n", __func__, err); 3867 3868 return err; 3869} 3870 3871/** 3872 * ufshcd_reset_and_restore - reset and re-initialize host/device 3873 * @hba: per-adapter instance 3874 * 3875 * Reset and recover device, host and re-establish link. This 3876 * is helpful to recover the communication in fatal error conditions. 3877 * 3878 * Returns zero on success, non-zero on failure 3879 */ 3880static int ufshcd_reset_and_restore(struct ufs_hba *hba) 3881{ 3882 int err = 0; 3883 unsigned long flags; 3884 int retries = MAX_HOST_RESET_RETRIES; 3885 3886 do { 3887 err = ufshcd_host_reset_and_restore(hba); 3888 } while (err && --retries); 3889 3890 /* 3891 * After reset the door-bell might be cleared, complete 3892 * outstanding requests in s/w here. 3893 */ 3894 spin_lock_irqsave(hba->host->host_lock, flags); 3895 ufshcd_transfer_req_compl(hba); 3896 ufshcd_tmc_handler(hba); 3897 spin_unlock_irqrestore(hba->host->host_lock, flags); 3898 3899 return err; 3900} 3901 3902/** 3903 * ufshcd_eh_host_reset_handler - host reset handler registered to scsi layer 3904 * @cmd - SCSI command pointer 3905 * 3906 * Returns SUCCESS/FAILED 3907 */ 3908static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd) 3909{ 3910 int err; 3911 unsigned long flags; 3912 struct ufs_hba *hba; 3913 3914 hba = shost_priv(cmd->device->host); 3915 3916 ufshcd_hold(hba, false); 3917 /* 3918 * Check if there is any race with fatal error handling. 3919 * If so, wait for it to complete. Even though fatal error 3920 * handling does reset and restore in some cases, don't assume 3921 * anything out of it. We are just avoiding race here. 3922 */ 3923 do { 3924 spin_lock_irqsave(hba->host->host_lock, flags); 3925 if (!(work_pending(&hba->eh_work) || 3926 hba->ufshcd_state == UFSHCD_STATE_RESET)) 3927 break; 3928 spin_unlock_irqrestore(hba->host->host_lock, flags); 3929 dev_dbg(hba->dev, "%s: reset in progress\n", __func__); 3930 flush_work(&hba->eh_work); 3931 } while (1); 3932 3933 hba->ufshcd_state = UFSHCD_STATE_RESET; 3934 ufshcd_set_eh_in_progress(hba); 3935 spin_unlock_irqrestore(hba->host->host_lock, flags); 3936 3937 err = ufshcd_reset_and_restore(hba); 3938 3939 spin_lock_irqsave(hba->host->host_lock, flags); 3940 if (!err) { 3941 err = SUCCESS; 3942 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL; 3943 } else { 3944 err = FAILED; 3945 hba->ufshcd_state = UFSHCD_STATE_ERROR; 3946 } 3947 ufshcd_clear_eh_in_progress(hba); 3948 spin_unlock_irqrestore(hba->host->host_lock, flags); 3949 3950 ufshcd_release(hba); 3951 return err; 3952} 3953 3954/** 3955 * ufshcd_get_max_icc_level - calculate the ICC level 3956 * @sup_curr_uA: max. current supported by the regulator 3957 * @start_scan: row at the desc table to start scan from 3958 * @buff: power descriptor buffer 3959 * 3960 * Returns calculated max ICC level for specific regulator 3961 */ 3962static u32 ufshcd_get_max_icc_level(int sup_curr_uA, u32 start_scan, char *buff) 3963{ 3964 int i; 3965 int curr_uA; 3966 u16 data; 3967 u16 unit; 3968 3969 for (i = start_scan; i >= 0; i--) { 3970 data = be16_to_cpu(*((u16 *)(buff + 2*i))); 3971 unit = (data & ATTR_ICC_LVL_UNIT_MASK) >> 3972 ATTR_ICC_LVL_UNIT_OFFSET; 3973 curr_uA = data & ATTR_ICC_LVL_VALUE_MASK; 3974 switch (unit) { 3975 case UFSHCD_NANO_AMP: 3976 curr_uA = curr_uA / 1000; 3977 break; 3978 case UFSHCD_MILI_AMP: 3979 curr_uA = curr_uA * 1000; 3980 break; 3981 case UFSHCD_AMP: 3982 curr_uA = curr_uA * 1000 * 1000; 3983 break; 3984 case UFSHCD_MICRO_AMP: 3985 default: 3986 break; 3987 } 3988 if (sup_curr_uA >= curr_uA) 3989 break; 3990 } 3991 if (i < 0) { 3992 i = 0; 3993 pr_err("%s: Couldn't find valid icc_level = %d", __func__, i); 3994 } 3995 3996 return (u32)i; 3997} 3998 3999/** 4000 * ufshcd_calc_icc_level - calculate the max ICC level 4001 * In case regulators are not initialized we'll return 0 4002 * @hba: per-adapter instance 4003 * @desc_buf: power descriptor buffer to extract ICC levels from. 4004 * @len: length of desc_buff 4005 * 4006 * Returns calculated ICC level 4007 */ 4008static u32 ufshcd_find_max_sup_active_icc_level(struct ufs_hba *hba, 4009 u8 *desc_buf, int len) 4010{ 4011 u32 icc_level = 0; 4012 4013 if (!hba->vreg_info.vcc || !hba->vreg_info.vccq || 4014 !hba->vreg_info.vccq2) { 4015 dev_err(hba->dev, 4016 "%s: Regulator capability was not set, actvIccLevel=%d", 4017 __func__, icc_level); 4018 goto out; 4019 } 4020 4021 if (hba->vreg_info.vcc) 4022 icc_level = ufshcd_get_max_icc_level( 4023 hba->vreg_info.vcc->max_uA, 4024 POWER_DESC_MAX_ACTV_ICC_LVLS - 1, 4025 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCC_0]); 4026 4027 if (hba->vreg_info.vccq) 4028 icc_level = ufshcd_get_max_icc_level( 4029 hba->vreg_info.vccq->max_uA, 4030 icc_level, 4031 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ_0]); 4032 4033 if (hba->vreg_info.vccq2) 4034 icc_level = ufshcd_get_max_icc_level( 4035 hba->vreg_info.vccq2->max_uA, 4036 icc_level, 4037 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ2_0]); 4038out: 4039 return icc_level; 4040} 4041 4042static void ufshcd_init_icc_levels(struct ufs_hba *hba) 4043{ 4044 int ret; 4045 int buff_len = QUERY_DESC_POWER_MAX_SIZE; 4046 u8 desc_buf[QUERY_DESC_POWER_MAX_SIZE]; 4047 4048 ret = ufshcd_read_power_desc(hba, desc_buf, buff_len); 4049 if (ret) { 4050 dev_err(hba->dev, 4051 "%s: Failed reading power descriptor.len = %d ret = %d", 4052 __func__, buff_len, ret); 4053 return; 4054 } 4055 4056 hba->init_prefetch_data.icc_level = 4057 ufshcd_find_max_sup_active_icc_level(hba, 4058 desc_buf, buff_len); 4059 dev_dbg(hba->dev, "%s: setting icc_level 0x%x", 4060 __func__, hba->init_prefetch_data.icc_level); 4061 4062 ret = ufshcd_query_attr(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, 4063 QUERY_ATTR_IDN_ACTIVE_ICC_LVL, 0, 0, 4064 &hba->init_prefetch_data.icc_level); 4065 4066 if (ret) 4067 dev_err(hba->dev, 4068 "%s: Failed configuring bActiveICCLevel = %d ret = %d", 4069 __func__, hba->init_prefetch_data.icc_level , ret); 4070 4071} 4072 4073/** 4074 * ufshcd_scsi_add_wlus - Adds required W-LUs 4075 * @hba: per-adapter instance 4076 * 4077 * UFS device specification requires the UFS devices to support 4 well known 4078 * logical units: 4079 * "REPORT_LUNS" (address: 01h) 4080 * "UFS Device" (address: 50h) 4081 * "RPMB" (address: 44h) 4082 * "BOOT" (address: 30h) 4083 * UFS device's power management needs to be controlled by "POWER CONDITION" 4084 * field of SSU (START STOP UNIT) command. But this "power condition" field 4085 * will take effect only when its sent to "UFS device" well known logical unit 4086 * hence we require the scsi_device instance to represent this logical unit in 4087 * order for the UFS host driver to send the SSU command for power management. 4088 4089 * We also require the scsi_device instance for "RPMB" (Replay Protected Memory 4090 * Block) LU so user space process can control this LU. User space may also 4091 * want to have access to BOOT LU. 4092 4093 * This function adds scsi device instances for each of all well known LUs 4094 * (except "REPORT LUNS" LU). 4095 * 4096 * Returns zero on success (all required W-LUs are added successfully), 4097 * non-zero error value on failure (if failed to add any of the required W-LU). 4098 */ 4099static int ufshcd_scsi_add_wlus(struct ufs_hba *hba) 4100{ 4101 int ret = 0; 4102 struct scsi_device *sdev_rpmb; 4103 struct scsi_device *sdev_boot; 4104 4105 hba->sdev_ufs_device = __scsi_add_device(hba->host, 0, 0, 4106 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_UFS_DEVICE_WLUN), NULL); 4107 if (IS_ERR(hba->sdev_ufs_device)) { 4108 ret = PTR_ERR(hba->sdev_ufs_device); 4109 hba->sdev_ufs_device = NULL; 4110 goto out; 4111 } 4112 scsi_device_put(hba->sdev_ufs_device); 4113 4114 sdev_boot = __scsi_add_device(hba->host, 0, 0, 4115 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_BOOT_WLUN), NULL); 4116 if (IS_ERR(sdev_boot)) { 4117 ret = PTR_ERR(sdev_boot); 4118 goto remove_sdev_ufs_device; 4119 } 4120 scsi_device_put(sdev_boot); 4121 4122 sdev_rpmb = __scsi_add_device(hba->host, 0, 0, 4123 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_RPMB_WLUN), NULL); 4124 if (IS_ERR(sdev_rpmb)) { 4125 ret = PTR_ERR(sdev_rpmb); 4126 goto remove_sdev_boot; 4127 } 4128 scsi_device_put(sdev_rpmb); 4129 goto out; 4130 4131remove_sdev_boot: 4132 scsi_remove_device(sdev_boot); 4133remove_sdev_ufs_device: 4134 scsi_remove_device(hba->sdev_ufs_device); 4135out: 4136 return ret; 4137} 4138 4139/** 4140 * ufshcd_probe_hba - probe hba to detect device and initialize 4141 * @hba: per-adapter instance 4142 * 4143 * Execute link-startup and verify device initialization 4144 */ 4145static int ufshcd_probe_hba(struct ufs_hba *hba) 4146{ 4147 int ret; 4148 4149 ret = ufshcd_link_startup(hba); 4150 if (ret) 4151 goto out; 4152 4153 ufshcd_init_pwr_info(hba); 4154 4155 /* UniPro link is active now */ 4156 ufshcd_set_link_active(hba); 4157 4158 ret = ufshcd_verify_dev_init(hba); 4159 if (ret) 4160 goto out; 4161 4162 ret = ufshcd_complete_dev_init(hba); 4163 if (ret) 4164 goto out; 4165 4166 /* UFS device is also active now */ 4167 ufshcd_set_ufs_dev_active(hba); 4168 ufshcd_force_reset_auto_bkops(hba); 4169 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL; 4170 hba->wlun_dev_clr_ua = true; 4171 4172 if (ufshcd_get_max_pwr_mode(hba)) { 4173 dev_err(hba->dev, 4174 "%s: Failed getting max supported power mode\n", 4175 __func__); 4176 } else { 4177 ret = ufshcd_config_pwr_mode(hba, &hba->max_pwr_info.info); 4178 if (ret) 4179 dev_err(hba->dev, "%s: Failed setting power mode, err = %d\n", 4180 __func__, ret); 4181 } 4182 4183 /* 4184 * If we are in error handling context or in power management callbacks 4185 * context, no need to scan the host 4186 */ 4187 if (!ufshcd_eh_in_progress(hba) && !hba->pm_op_in_progress) { 4188 bool flag; 4189 4190 /* clear any previous UFS device information */ 4191 memset(&hba->dev_info, 0, sizeof(hba->dev_info)); 4192 if (!ufshcd_query_flag(hba, UPIU_QUERY_OPCODE_READ_FLAG, 4193 QUERY_FLAG_IDN_PWR_ON_WPE, &flag)) 4194 hba->dev_info.f_power_on_wp_en = flag; 4195 4196 if (!hba->is_init_prefetch) 4197 ufshcd_init_icc_levels(hba); 4198 4199 /* Add required well known logical units to scsi mid layer */ 4200 if (ufshcd_scsi_add_wlus(hba)) 4201 goto out; 4202 4203 scsi_scan_host(hba->host); 4204 pm_runtime_put_sync(hba->dev); 4205 } 4206 4207 if (!hba->is_init_prefetch) 4208 hba->is_init_prefetch = true; 4209 4210 /* Resume devfreq after UFS device is detected */ 4211 if (ufshcd_is_clkscaling_enabled(hba)) 4212 devfreq_resume_device(hba->devfreq); 4213 4214out: 4215 /* 4216 * If we failed to initialize the device or the device is not 4217 * present, turn off the power/clocks etc. 4218 */ 4219 if (ret && !ufshcd_eh_in_progress(hba) && !hba->pm_op_in_progress) { 4220 pm_runtime_put_sync(hba->dev); 4221 ufshcd_hba_exit(hba); 4222 } 4223 4224 return ret; 4225} 4226 4227/** 4228 * ufshcd_async_scan - asynchronous execution for probing hba 4229 * @data: data pointer to pass to this function 4230 * @cookie: cookie data 4231 */ 4232static void ufshcd_async_scan(void *data, async_cookie_t cookie) 4233{ 4234 struct ufs_hba *hba = (struct ufs_hba *)data; 4235 4236 ufshcd_probe_hba(hba); 4237} 4238 4239static struct scsi_host_template ufshcd_driver_template = { 4240 .module = THIS_MODULE, 4241 .name = UFSHCD, 4242 .proc_name = UFSHCD, 4243 .queuecommand = ufshcd_queuecommand, 4244 .slave_alloc = ufshcd_slave_alloc, 4245 .slave_configure = ufshcd_slave_configure, 4246 .slave_destroy = ufshcd_slave_destroy, 4247 .change_queue_depth = ufshcd_change_queue_depth, 4248 .eh_abort_handler = ufshcd_abort, 4249 .eh_device_reset_handler = ufshcd_eh_device_reset_handler, 4250 .eh_host_reset_handler = ufshcd_eh_host_reset_handler, 4251 .this_id = -1, 4252 .sg_tablesize = SG_ALL, 4253 .cmd_per_lun = UFSHCD_CMD_PER_LUN, 4254 .can_queue = UFSHCD_CAN_QUEUE, 4255 .max_host_blocked = 1, 4256 .use_blk_tags = 1, 4257 .track_queue_depth = 1, 4258}; 4259 4260static int ufshcd_config_vreg_load(struct device *dev, struct ufs_vreg *vreg, 4261 int ua) 4262{ 4263 int ret; 4264 4265 if (!vreg) 4266 return 0; 4267 4268 ret = regulator_set_load(vreg->reg, ua); 4269 if (ret < 0) { 4270 dev_err(dev, "%s: %s set load (ua=%d) failed, err=%d\n", 4271 __func__, vreg->name, ua, ret); 4272 } 4273 4274 return ret; 4275} 4276 4277static inline int ufshcd_config_vreg_lpm(struct ufs_hba *hba, 4278 struct ufs_vreg *vreg) 4279{ 4280 return ufshcd_config_vreg_load(hba->dev, vreg, UFS_VREG_LPM_LOAD_UA); 4281} 4282 4283static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba, 4284 struct ufs_vreg *vreg) 4285{ 4286 return ufshcd_config_vreg_load(hba->dev, vreg, vreg->max_uA); 4287} 4288 4289static int ufshcd_config_vreg(struct device *dev, 4290 struct ufs_vreg *vreg, bool on) 4291{ 4292 int ret = 0; 4293 struct regulator *reg = vreg->reg; 4294 const char *name = vreg->name; 4295 int min_uV, uA_load; 4296 4297 BUG_ON(!vreg); 4298 4299 if (regulator_count_voltages(reg) > 0) { 4300 min_uV = on ? vreg->min_uV : 0; 4301 ret = regulator_set_voltage(reg, min_uV, vreg->max_uV); 4302 if (ret) { 4303 dev_err(dev, "%s: %s set voltage failed, err=%d\n", 4304 __func__, name, ret); 4305 goto out; 4306 } 4307 4308 uA_load = on ? vreg->max_uA : 0; 4309 ret = ufshcd_config_vreg_load(dev, vreg, uA_load); 4310 if (ret) 4311 goto out; 4312 } 4313out: 4314 return ret; 4315} 4316 4317static int ufshcd_enable_vreg(struct device *dev, struct ufs_vreg *vreg) 4318{ 4319 int ret = 0; 4320 4321 if (!vreg || vreg->enabled) 4322 goto out; 4323 4324 ret = ufshcd_config_vreg(dev, vreg, true); 4325 if (!ret) 4326 ret = regulator_enable(vreg->reg); 4327 4328 if (!ret) 4329 vreg->enabled = true; 4330 else 4331 dev_err(dev, "%s: %s enable failed, err=%d\n", 4332 __func__, vreg->name, ret); 4333out: 4334 return ret; 4335} 4336 4337static int ufshcd_disable_vreg(struct device *dev, struct ufs_vreg *vreg) 4338{ 4339 int ret = 0; 4340 4341 if (!vreg || !vreg->enabled) 4342 goto out; 4343 4344 ret = regulator_disable(vreg->reg); 4345 4346 if (!ret) { 4347 /* ignore errors on applying disable config */ 4348 ufshcd_config_vreg(dev, vreg, false); 4349 vreg->enabled = false; 4350 } else { 4351 dev_err(dev, "%s: %s disable failed, err=%d\n", 4352 __func__, vreg->name, ret); 4353 } 4354out: 4355 return ret; 4356} 4357 4358static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on) 4359{ 4360 int ret = 0; 4361 struct device *dev = hba->dev; 4362 struct ufs_vreg_info *info = &hba->vreg_info; 4363 4364 if (!info) 4365 goto out; 4366 4367 ret = ufshcd_toggle_vreg(dev, info->vcc, on); 4368 if (ret) 4369 goto out; 4370 4371 ret = ufshcd_toggle_vreg(dev, info->vccq, on); 4372 if (ret) 4373 goto out; 4374 4375 ret = ufshcd_toggle_vreg(dev, info->vccq2, on); 4376 if (ret) 4377 goto out; 4378 4379out: 4380 if (ret) { 4381 ufshcd_toggle_vreg(dev, info->vccq2, false); 4382 ufshcd_toggle_vreg(dev, info->vccq, false); 4383 ufshcd_toggle_vreg(dev, info->vcc, false); 4384 } 4385 return ret; 4386} 4387 4388static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on) 4389{ 4390 struct ufs_vreg_info *info = &hba->vreg_info; 4391 4392 if (info) 4393 return ufshcd_toggle_vreg(hba->dev, info->vdd_hba, on); 4394 4395 return 0; 4396} 4397 4398static int ufshcd_get_vreg(struct device *dev, struct ufs_vreg *vreg) 4399{ 4400 int ret = 0; 4401 4402 if (!vreg) 4403 goto out; 4404 4405 vreg->reg = devm_regulator_get(dev, vreg->name); 4406 if (IS_ERR(vreg->reg)) { 4407 ret = PTR_ERR(vreg->reg); 4408 dev_err(dev, "%s: %s get failed, err=%d\n", 4409 __func__, vreg->name, ret); 4410 } 4411out: 4412 return ret; 4413} 4414 4415static int ufshcd_init_vreg(struct ufs_hba *hba) 4416{ 4417 int ret = 0; 4418 struct device *dev = hba->dev; 4419 struct ufs_vreg_info *info = &hba->vreg_info; 4420 4421 if (!info) 4422 goto out; 4423 4424 ret = ufshcd_get_vreg(dev, info->vcc); 4425 if (ret) 4426 goto out; 4427 4428 ret = ufshcd_get_vreg(dev, info->vccq); 4429 if (ret) 4430 goto out; 4431 4432 ret = ufshcd_get_vreg(dev, info->vccq2); 4433out: 4434 return ret; 4435} 4436 4437static int ufshcd_init_hba_vreg(struct ufs_hba *hba) 4438{ 4439 struct ufs_vreg_info *info = &hba->vreg_info; 4440 4441 if (info) 4442 return ufshcd_get_vreg(hba->dev, info->vdd_hba); 4443 4444 return 0; 4445} 4446 4447static int __ufshcd_setup_clocks(struct ufs_hba *hba, bool on, 4448 bool skip_ref_clk) 4449{ 4450 int ret = 0; 4451 struct ufs_clk_info *clki; 4452 struct list_head *head = &hba->clk_list_head; 4453 unsigned long flags; 4454 4455 if (!head || list_empty(head)) 4456 goto out; 4457 4458 list_for_each_entry(clki, head, list) { 4459 if (!IS_ERR_OR_NULL(clki->clk)) { 4460 if (skip_ref_clk && !strcmp(clki->name, "ref_clk")) 4461 continue; 4462 4463 if (on && !clki->enabled) { 4464 ret = clk_prepare_enable(clki->clk); 4465 if (ret) { 4466 dev_err(hba->dev, "%s: %s prepare enable failed, %d\n", 4467 __func__, clki->name, ret); 4468 goto out; 4469 } 4470 } else if (!on && clki->enabled) { 4471 clk_disable_unprepare(clki->clk); 4472 } 4473 clki->enabled = on; 4474 dev_dbg(hba->dev, "%s: clk: %s %sabled\n", __func__, 4475 clki->name, on ? "en" : "dis"); 4476 } 4477 } 4478 4479 if (hba->vops && hba->vops->setup_clocks) 4480 ret = hba->vops->setup_clocks(hba, on); 4481out: 4482 if (ret) { 4483 list_for_each_entry(clki, head, list) { 4484 if (!IS_ERR_OR_NULL(clki->clk) && clki->enabled) 4485 clk_disable_unprepare(clki->clk); 4486 } 4487 } else if (on) { 4488 spin_lock_irqsave(hba->host->host_lock, flags); 4489 hba->clk_gating.state = CLKS_ON; 4490 spin_unlock_irqrestore(hba->host->host_lock, flags); 4491 } 4492 return ret; 4493} 4494 4495static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on) 4496{ 4497 return __ufshcd_setup_clocks(hba, on, false); 4498} 4499 4500static int ufshcd_init_clocks(struct ufs_hba *hba) 4501{ 4502 int ret = 0; 4503 struct ufs_clk_info *clki; 4504 struct device *dev = hba->dev; 4505 struct list_head *head = &hba->clk_list_head; 4506 4507 if (!head || list_empty(head)) 4508 goto out; 4509 4510 list_for_each_entry(clki, head, list) { 4511 if (!clki->name) 4512 continue; 4513 4514 clki->clk = devm_clk_get(dev, clki->name); 4515 if (IS_ERR(clki->clk)) { 4516 ret = PTR_ERR(clki->clk); 4517 dev_err(dev, "%s: %s clk get failed, %d\n", 4518 __func__, clki->name, ret); 4519 goto out; 4520 } 4521 4522 if (clki->max_freq) { 4523 ret = clk_set_rate(clki->clk, clki->max_freq); 4524 if (ret) { 4525 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n", 4526 __func__, clki->name, 4527 clki->max_freq, ret); 4528 goto out; 4529 } 4530 clki->curr_freq = clki->max_freq; 4531 } 4532 dev_dbg(dev, "%s: clk: %s, rate: %lu\n", __func__, 4533 clki->name, clk_get_rate(clki->clk)); 4534 } 4535out: 4536 return ret; 4537} 4538 4539static int ufshcd_variant_hba_init(struct ufs_hba *hba) 4540{ 4541 int err = 0; 4542 4543 if (!hba->vops) 4544 goto out; 4545 4546 if (hba->vops->init) { 4547 err = hba->vops->init(hba); 4548 if (err) 4549 goto out; 4550 } 4551 4552 if (hba->vops->setup_regulators) { 4553 err = hba->vops->setup_regulators(hba, true); 4554 if (err) 4555 goto out_exit; 4556 } 4557 4558 goto out; 4559 4560out_exit: 4561 if (hba->vops->exit) 4562 hba->vops->exit(hba); 4563out: 4564 if (err) 4565 dev_err(hba->dev, "%s: variant %s init failed err %d\n", 4566 __func__, hba->vops ? hba->vops->name : "", err); 4567 return err; 4568} 4569 4570static void ufshcd_variant_hba_exit(struct ufs_hba *hba) 4571{ 4572 if (!hba->vops) 4573 return; 4574 4575 if (hba->vops->setup_clocks) 4576 hba->vops->setup_clocks(hba, false); 4577 4578 if (hba->vops->setup_regulators) 4579 hba->vops->setup_regulators(hba, false); 4580 4581 if (hba->vops->exit) 4582 hba->vops->exit(hba); 4583} 4584 4585static int ufshcd_hba_init(struct ufs_hba *hba) 4586{ 4587 int err; 4588 4589 /* 4590 * Handle host controller power separately from the UFS device power 4591 * rails as it will help controlling the UFS host controller power 4592 * collapse easily which is different than UFS device power collapse. 4593 * Also, enable the host controller power before we go ahead with rest 4594 * of the initialization here. 4595 */ 4596 err = ufshcd_init_hba_vreg(hba); 4597 if (err) 4598 goto out; 4599 4600 err = ufshcd_setup_hba_vreg(hba, true); 4601 if (err) 4602 goto out; 4603 4604 err = ufshcd_init_clocks(hba); 4605 if (err) 4606 goto out_disable_hba_vreg; 4607 4608 err = ufshcd_setup_clocks(hba, true); 4609 if (err) 4610 goto out_disable_hba_vreg; 4611 4612 err = ufshcd_init_vreg(hba); 4613 if (err) 4614 goto out_disable_clks; 4615 4616 err = ufshcd_setup_vreg(hba, true); 4617 if (err) 4618 goto out_disable_clks; 4619 4620 err = ufshcd_variant_hba_init(hba); 4621 if (err) 4622 goto out_disable_vreg; 4623 4624 hba->is_powered = true; 4625 goto out; 4626 4627out_disable_vreg: 4628 ufshcd_setup_vreg(hba, false); 4629out_disable_clks: 4630 ufshcd_setup_clocks(hba, false); 4631out_disable_hba_vreg: 4632 ufshcd_setup_hba_vreg(hba, false); 4633out: 4634 return err; 4635} 4636 4637static void ufshcd_hba_exit(struct ufs_hba *hba) 4638{ 4639 if (hba->is_powered) { 4640 ufshcd_variant_hba_exit(hba); 4641 ufshcd_setup_vreg(hba, false); 4642 ufshcd_setup_clocks(hba, false); 4643 ufshcd_setup_hba_vreg(hba, false); 4644 hba->is_powered = false; 4645 } 4646} 4647 4648static int 4649ufshcd_send_request_sense(struct ufs_hba *hba, struct scsi_device *sdp) 4650{ 4651 unsigned char cmd[6] = {REQUEST_SENSE, 4652 0, 4653 0, 4654 0, 4655 SCSI_SENSE_BUFFERSIZE, 4656 0}; 4657 char *buffer; 4658 int ret; 4659 4660 buffer = kzalloc(SCSI_SENSE_BUFFERSIZE, GFP_KERNEL); 4661 if (!buffer) { 4662 ret = -ENOMEM; 4663 goto out; 4664 } 4665 4666 ret = scsi_execute_req_flags(sdp, cmd, DMA_FROM_DEVICE, buffer, 4667 SCSI_SENSE_BUFFERSIZE, NULL, 4668 msecs_to_jiffies(1000), 3, NULL, REQ_PM); 4669 if (ret) 4670 pr_err("%s: failed with err %d\n", __func__, ret); 4671 4672 kfree(buffer); 4673out: 4674 return ret; 4675} 4676 4677/** 4678 * ufshcd_set_dev_pwr_mode - sends START STOP UNIT command to set device 4679 * power mode 4680 * @hba: per adapter instance 4681 * @pwr_mode: device power mode to set 4682 * 4683 * Returns 0 if requested power mode is set successfully 4684 * Returns non-zero if failed to set the requested power mode 4685 */ 4686static int ufshcd_set_dev_pwr_mode(struct ufs_hba *hba, 4687 enum ufs_dev_pwr_mode pwr_mode) 4688{ 4689 unsigned char cmd[6] = { START_STOP }; 4690 struct scsi_sense_hdr sshdr; 4691 struct scsi_device *sdp; 4692 unsigned long flags; 4693 int ret; 4694 4695 spin_lock_irqsave(hba->host->host_lock, flags); 4696 sdp = hba->sdev_ufs_device; 4697 if (sdp) { 4698 ret = scsi_device_get(sdp); 4699 if (!ret && !scsi_device_online(sdp)) { 4700 ret = -ENODEV; 4701 scsi_device_put(sdp); 4702 } 4703 } else { 4704 ret = -ENODEV; 4705 } 4706 spin_unlock_irqrestore(hba->host->host_lock, flags); 4707 4708 if (ret) 4709 return ret; 4710 4711 /* 4712 * If scsi commands fail, the scsi mid-layer schedules scsi error- 4713 * handling, which would wait for host to be resumed. Since we know 4714 * we are functional while we are here, skip host resume in error 4715 * handling context. 4716 */ 4717 hba->host->eh_noresume = 1; 4718 if (hba->wlun_dev_clr_ua) { 4719 ret = ufshcd_send_request_sense(hba, sdp); 4720 if (ret) 4721 goto out; 4722 /* Unit attention condition is cleared now */ 4723 hba->wlun_dev_clr_ua = false; 4724 } 4725 4726 cmd[4] = pwr_mode << 4; 4727 4728 /* 4729 * Current function would be generally called from the power management 4730 * callbacks hence set the REQ_PM flag so that it doesn't resume the 4731 * already suspended childs. 4732 */ 4733 ret = scsi_execute_req_flags(sdp, cmd, DMA_NONE, NULL, 0, &sshdr, 4734 START_STOP_TIMEOUT, 0, NULL, REQ_PM); 4735 if (ret) { 4736 sdev_printk(KERN_WARNING, sdp, 4737 "START_STOP failed for power mode: %d, result %x\n", 4738 pwr_mode, ret); 4739 if (driver_byte(ret) & DRIVER_SENSE) 4740 scsi_print_sense_hdr(sdp, NULL, &sshdr); 4741 } 4742 4743 if (!ret) 4744 hba->curr_dev_pwr_mode = pwr_mode; 4745out: 4746 scsi_device_put(sdp); 4747 hba->host->eh_noresume = 0; 4748 return ret; 4749} 4750 4751static int ufshcd_link_state_transition(struct ufs_hba *hba, 4752 enum uic_link_state req_link_state, 4753 int check_for_bkops) 4754{ 4755 int ret = 0; 4756 4757 if (req_link_state == hba->uic_link_state) 4758 return 0; 4759 4760 if (req_link_state == UIC_LINK_HIBERN8_STATE) { 4761 ret = ufshcd_uic_hibern8_enter(hba); 4762 if (!ret) 4763 ufshcd_set_link_hibern8(hba); 4764 else 4765 goto out; 4766 } 4767 /* 4768 * If autobkops is enabled, link can't be turned off because 4769 * turning off the link would also turn off the device. 4770 */ 4771 else if ((req_link_state == UIC_LINK_OFF_STATE) && 4772 (!check_for_bkops || (check_for_bkops && 4773 !hba->auto_bkops_enabled))) { 4774 /* 4775 * Change controller state to "reset state" which 4776 * should also put the link in off/reset state 4777 */ 4778 ufshcd_hba_stop(hba); 4779 /* 4780 * TODO: Check if we need any delay to make sure that 4781 * controller is reset 4782 */ 4783 ufshcd_set_link_off(hba); 4784 } 4785 4786out: 4787 return ret; 4788} 4789 4790static void ufshcd_vreg_set_lpm(struct ufs_hba *hba) 4791{ 4792 /* 4793 * If UFS device is either in UFS_Sleep turn off VCC rail to save some 4794 * power. 4795 * 4796 * If UFS device and link is in OFF state, all power supplies (VCC, 4797 * VCCQ, VCCQ2) can be turned off if power on write protect is not 4798 * required. If UFS link is inactive (Hibern8 or OFF state) and device 4799 * is in sleep state, put VCCQ & VCCQ2 rails in LPM mode. 4800 * 4801 * Ignore the error returned by ufshcd_toggle_vreg() as device is anyway 4802 * in low power state which would save some power. 4803 */ 4804 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) && 4805 !hba->dev_info.is_lu_power_on_wp) { 4806 ufshcd_setup_vreg(hba, false); 4807 } else if (!ufshcd_is_ufs_dev_active(hba)) { 4808 ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false); 4809 if (!ufshcd_is_link_active(hba)) { 4810 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq); 4811 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq2); 4812 } 4813 } 4814} 4815 4816static int ufshcd_vreg_set_hpm(struct ufs_hba *hba) 4817{ 4818 int ret = 0; 4819 4820 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) && 4821 !hba->dev_info.is_lu_power_on_wp) { 4822 ret = ufshcd_setup_vreg(hba, true); 4823 } else if (!ufshcd_is_ufs_dev_active(hba)) { 4824 ret = ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, true); 4825 if (!ret && !ufshcd_is_link_active(hba)) { 4826 ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq); 4827 if (ret) 4828 goto vcc_disable; 4829 ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2); 4830 if (ret) 4831 goto vccq_lpm; 4832 } 4833 } 4834 goto out; 4835 4836vccq_lpm: 4837 ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq); 4838vcc_disable: 4839 ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false); 4840out: 4841 return ret; 4842} 4843 4844static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba) 4845{ 4846 if (ufshcd_is_link_off(hba)) 4847 ufshcd_setup_hba_vreg(hba, false); 4848} 4849 4850static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba) 4851{ 4852 if (ufshcd_is_link_off(hba)) 4853 ufshcd_setup_hba_vreg(hba, true); 4854} 4855 4856/** 4857 * ufshcd_suspend - helper function for suspend operations 4858 * @hba: per adapter instance 4859 * @pm_op: desired low power operation type 4860 * 4861 * This function will try to put the UFS device and link into low power 4862 * mode based on the "rpm_lvl" (Runtime PM level) or "spm_lvl" 4863 * (System PM level). 4864 * 4865 * If this function is called during shutdown, it will make sure that 4866 * both UFS device and UFS link is powered off. 4867 * 4868 * NOTE: UFS device & link must be active before we enter in this function. 4869 * 4870 * Returns 0 for success and non-zero for failure 4871 */ 4872static int ufshcd_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op) 4873{ 4874 int ret = 0; 4875 enum ufs_pm_level pm_lvl; 4876 enum ufs_dev_pwr_mode req_dev_pwr_mode; 4877 enum uic_link_state req_link_state; 4878 4879 hba->pm_op_in_progress = 1; 4880 if (!ufshcd_is_shutdown_pm(pm_op)) { 4881 pm_lvl = ufshcd_is_runtime_pm(pm_op) ? 4882 hba->rpm_lvl : hba->spm_lvl; 4883 req_dev_pwr_mode = ufs_get_pm_lvl_to_dev_pwr_mode(pm_lvl); 4884 req_link_state = ufs_get_pm_lvl_to_link_pwr_state(pm_lvl); 4885 } else { 4886 req_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE; 4887 req_link_state = UIC_LINK_OFF_STATE; 4888 } 4889 4890 /* 4891 * If we can't transition into any of the low power modes 4892 * just gate the clocks. 4893 */ 4894 ufshcd_hold(hba, false); 4895 hba->clk_gating.is_suspended = true; 4896 4897 if (req_dev_pwr_mode == UFS_ACTIVE_PWR_MODE && 4898 req_link_state == UIC_LINK_ACTIVE_STATE) { 4899 goto disable_clks; 4900 } 4901 4902 if ((req_dev_pwr_mode == hba->curr_dev_pwr_mode) && 4903 (req_link_state == hba->uic_link_state)) 4904 goto out; 4905 4906 /* UFS device & link must be active before we enter in this function */ 4907 if (!ufshcd_is_ufs_dev_active(hba) || !ufshcd_is_link_active(hba)) { 4908 ret = -EINVAL; 4909 goto out; 4910 } 4911 4912 if (ufshcd_is_runtime_pm(pm_op)) { 4913 if (ufshcd_can_autobkops_during_suspend(hba)) { 4914 /* 4915 * The device is idle with no requests in the queue, 4916 * allow background operations if bkops status shows 4917 * that performance might be impacted. 4918 */ 4919 ret = ufshcd_urgent_bkops(hba); 4920 if (ret) 4921 goto enable_gating; 4922 } else { 4923 /* make sure that auto bkops is disabled */ 4924 ufshcd_disable_auto_bkops(hba); 4925 } 4926 } 4927 4928 if ((req_dev_pwr_mode != hba->curr_dev_pwr_mode) && 4929 ((ufshcd_is_runtime_pm(pm_op) && !hba->auto_bkops_enabled) || 4930 !ufshcd_is_runtime_pm(pm_op))) { 4931 /* ensure that bkops is disabled */ 4932 ufshcd_disable_auto_bkops(hba); 4933 ret = ufshcd_set_dev_pwr_mode(hba, req_dev_pwr_mode); 4934 if (ret) 4935 goto enable_gating; 4936 } 4937 4938 ret = ufshcd_link_state_transition(hba, req_link_state, 1); 4939 if (ret) 4940 goto set_dev_active; 4941 4942 ufshcd_vreg_set_lpm(hba); 4943 4944disable_clks: 4945 /* 4946 * The clock scaling needs access to controller registers. Hence, Wait 4947 * for pending clock scaling work to be done before clocks are 4948 * turned off. 4949 */ 4950 if (ufshcd_is_clkscaling_enabled(hba)) { 4951 devfreq_suspend_device(hba->devfreq); 4952 hba->clk_scaling.window_start_t = 0; 4953 } 4954 /* 4955 * Call vendor specific suspend callback. As these callbacks may access 4956 * vendor specific host controller register space call them before the 4957 * host clocks are ON. 4958 */ 4959 if (hba->vops && hba->vops->suspend) { 4960 ret = hba->vops->suspend(hba, pm_op); 4961 if (ret) 4962 goto set_link_active; 4963 } 4964 4965 if (hba->vops && hba->vops->setup_clocks) { 4966 ret = hba->vops->setup_clocks(hba, false); 4967 if (ret) 4968 goto vops_resume; 4969 } 4970 4971 if (!ufshcd_is_link_active(hba)) 4972 ufshcd_setup_clocks(hba, false); 4973 else 4974 /* If link is active, device ref_clk can't be switched off */ 4975 __ufshcd_setup_clocks(hba, false, true); 4976 4977 hba->clk_gating.state = CLKS_OFF; 4978 /* 4979 * Disable the host irq as host controller as there won't be any 4980 * host controller trasanction expected till resume. 4981 */ 4982 ufshcd_disable_irq(hba); 4983 /* Put the host controller in low power mode if possible */ 4984 ufshcd_hba_vreg_set_lpm(hba); 4985 goto out; 4986 4987vops_resume: 4988 if (hba->vops && hba->vops->resume) 4989 hba->vops->resume(hba, pm_op); 4990set_link_active: 4991 ufshcd_vreg_set_hpm(hba); 4992 if (ufshcd_is_link_hibern8(hba) && !ufshcd_uic_hibern8_exit(hba)) 4993 ufshcd_set_link_active(hba); 4994 else if (ufshcd_is_link_off(hba)) 4995 ufshcd_host_reset_and_restore(hba); 4996set_dev_active: 4997 if (!ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE)) 4998 ufshcd_disable_auto_bkops(hba); 4999enable_gating: 5000 hba->clk_gating.is_suspended = false; 5001 ufshcd_release(hba); 5002out: 5003 hba->pm_op_in_progress = 0; 5004 return ret; 5005} 5006 5007/** 5008 * ufshcd_resume - helper function for resume operations 5009 * @hba: per adapter instance 5010 * @pm_op: runtime PM or system PM 5011 * 5012 * This function basically brings the UFS device, UniPro link and controller 5013 * to active state. 5014 * 5015 * Returns 0 for success and non-zero for failure 5016 */ 5017static int ufshcd_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op) 5018{ 5019 int ret; 5020 enum uic_link_state old_link_state; 5021 5022 hba->pm_op_in_progress = 1; 5023 old_link_state = hba->uic_link_state; 5024 5025 ufshcd_hba_vreg_set_hpm(hba); 5026 /* Make sure clocks are enabled before accessing controller */ 5027 ret = ufshcd_setup_clocks(hba, true); 5028 if (ret) 5029 goto out; 5030 5031 /* enable the host irq as host controller would be active soon */ 5032 ret = ufshcd_enable_irq(hba); 5033 if (ret) 5034 goto disable_irq_and_vops_clks; 5035 5036 ret = ufshcd_vreg_set_hpm(hba); 5037 if (ret) 5038 goto disable_irq_and_vops_clks; 5039 5040 /* 5041 * Call vendor specific resume callback. As these callbacks may access 5042 * vendor specific host controller register space call them when the 5043 * host clocks are ON. 5044 */ 5045 if (hba->vops && hba->vops->resume) { 5046 ret = hba->vops->resume(hba, pm_op); 5047 if (ret) 5048 goto disable_vreg; 5049 } 5050 5051 if (ufshcd_is_link_hibern8(hba)) { 5052 ret = ufshcd_uic_hibern8_exit(hba); 5053 if (!ret) 5054 ufshcd_set_link_active(hba); 5055 else 5056 goto vendor_suspend; 5057 } else if (ufshcd_is_link_off(hba)) { 5058 ret = ufshcd_host_reset_and_restore(hba); 5059 /* 5060 * ufshcd_host_reset_and_restore() should have already 5061 * set the link state as active 5062 */ 5063 if (ret || !ufshcd_is_link_active(hba)) 5064 goto vendor_suspend; 5065 } 5066 5067 if (!ufshcd_is_ufs_dev_active(hba)) { 5068 ret = ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE); 5069 if (ret) 5070 goto set_old_link_state; 5071 } 5072 5073 /* 5074 * If BKOPs operations are urgently needed at this moment then 5075 * keep auto-bkops enabled or else disable it. 5076 */ 5077 ufshcd_urgent_bkops(hba); 5078 hba->clk_gating.is_suspended = false; 5079 5080 if (ufshcd_is_clkscaling_enabled(hba)) 5081 devfreq_resume_device(hba->devfreq); 5082 5083 /* Schedule clock gating in case of no access to UFS device yet */ 5084 ufshcd_release(hba); 5085 goto out; 5086 5087set_old_link_state: 5088 ufshcd_link_state_transition(hba, old_link_state, 0); 5089vendor_suspend: 5090 if (hba->vops && hba->vops->suspend) 5091 hba->vops->suspend(hba, pm_op); 5092disable_vreg: 5093 ufshcd_vreg_set_lpm(hba); 5094disable_irq_and_vops_clks: 5095 ufshcd_disable_irq(hba); 5096 ufshcd_setup_clocks(hba, false); 5097out: 5098 hba->pm_op_in_progress = 0; 5099 return ret; 5100} 5101 5102/** 5103 * ufshcd_system_suspend - system suspend routine 5104 * @hba: per adapter instance 5105 * @pm_op: runtime PM or system PM 5106 * 5107 * Check the description of ufshcd_suspend() function for more details. 5108 * 5109 * Returns 0 for success and non-zero for failure 5110 */ 5111int ufshcd_system_suspend(struct ufs_hba *hba) 5112{ 5113 int ret = 0; 5114 5115 if (!hba || !hba->is_powered) 5116 return 0; 5117 5118 if (pm_runtime_suspended(hba->dev)) { 5119 if (hba->rpm_lvl == hba->spm_lvl) 5120 /* 5121 * There is possibility that device may still be in 5122 * active state during the runtime suspend. 5123 */ 5124 if ((ufs_get_pm_lvl_to_dev_pwr_mode(hba->spm_lvl) == 5125 hba->curr_dev_pwr_mode) && !hba->auto_bkops_enabled) 5126 goto out; 5127 5128 /* 5129 * UFS device and/or UFS link low power states during runtime 5130 * suspend seems to be different than what is expected during 5131 * system suspend. Hence runtime resume the devic & link and 5132 * let the system suspend low power states to take effect. 5133 * TODO: If resume takes longer time, we might have optimize 5134 * it in future by not resuming everything if possible. 5135 */ 5136 ret = ufshcd_runtime_resume(hba); 5137 if (ret) 5138 goto out; 5139 } 5140 5141 ret = ufshcd_suspend(hba, UFS_SYSTEM_PM); 5142out: 5143 if (!ret) 5144 hba->is_sys_suspended = true; 5145 return ret; 5146} 5147EXPORT_SYMBOL(ufshcd_system_suspend); 5148 5149/** 5150 * ufshcd_system_resume - system resume routine 5151 * @hba: per adapter instance 5152 * 5153 * Returns 0 for success and non-zero for failure 5154 */ 5155 5156int ufshcd_system_resume(struct ufs_hba *hba) 5157{ 5158 if (!hba || !hba->is_powered || pm_runtime_suspended(hba->dev)) 5159 /* 5160 * Let the runtime resume take care of resuming 5161 * if runtime suspended. 5162 */ 5163 return 0; 5164 5165 return ufshcd_resume(hba, UFS_SYSTEM_PM); 5166} 5167EXPORT_SYMBOL(ufshcd_system_resume); 5168 5169/** 5170 * ufshcd_runtime_suspend - runtime suspend routine 5171 * @hba: per adapter instance 5172 * 5173 * Check the description of ufshcd_suspend() function for more details. 5174 * 5175 * Returns 0 for success and non-zero for failure 5176 */ 5177int ufshcd_runtime_suspend(struct ufs_hba *hba) 5178{ 5179 if (!hba || !hba->is_powered) 5180 return 0; 5181 5182 return ufshcd_suspend(hba, UFS_RUNTIME_PM); 5183} 5184EXPORT_SYMBOL(ufshcd_runtime_suspend); 5185 5186/** 5187 * ufshcd_runtime_resume - runtime resume routine 5188 * @hba: per adapter instance 5189 * 5190 * This function basically brings the UFS device, UniPro link and controller 5191 * to active state. Following operations are done in this function: 5192 * 5193 * 1. Turn on all the controller related clocks 5194 * 2. Bring the UniPro link out of Hibernate state 5195 * 3. If UFS device is in sleep state, turn ON VCC rail and bring the UFS device 5196 * to active state. 5197 * 4. If auto-bkops is enabled on the device, disable it. 5198 * 5199 * So following would be the possible power state after this function return 5200 * successfully: 5201 * S1: UFS device in Active state with VCC rail ON 5202 * UniPro link in Active state 5203 * All the UFS/UniPro controller clocks are ON 5204 * 5205 * Returns 0 for success and non-zero for failure 5206 */ 5207int ufshcd_runtime_resume(struct ufs_hba *hba) 5208{ 5209 if (!hba || !hba->is_powered) 5210 return 0; 5211 else 5212 return ufshcd_resume(hba, UFS_RUNTIME_PM); 5213} 5214EXPORT_SYMBOL(ufshcd_runtime_resume); 5215 5216int ufshcd_runtime_idle(struct ufs_hba *hba) 5217{ 5218 return 0; 5219} 5220EXPORT_SYMBOL(ufshcd_runtime_idle); 5221 5222/** 5223 * ufshcd_shutdown - shutdown routine 5224 * @hba: per adapter instance 5225 * 5226 * This function would power off both UFS device and UFS link. 5227 * 5228 * Returns 0 always to allow force shutdown even in case of errors. 5229 */ 5230int ufshcd_shutdown(struct ufs_hba *hba) 5231{ 5232 int ret = 0; 5233 5234 if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba)) 5235 goto out; 5236 5237 if (pm_runtime_suspended(hba->dev)) { 5238 ret = ufshcd_runtime_resume(hba); 5239 if (ret) 5240 goto out; 5241 } 5242 5243 ret = ufshcd_suspend(hba, UFS_SHUTDOWN_PM); 5244out: 5245 if (ret) 5246 dev_err(hba->dev, "%s failed, err %d\n", __func__, ret); 5247 /* allow force shutdown even in case of errors */ 5248 return 0; 5249} 5250EXPORT_SYMBOL(ufshcd_shutdown); 5251 5252/** 5253 * ufshcd_remove - de-allocate SCSI host and host memory space 5254 * data structure memory 5255 * @hba - per adapter instance 5256 */ 5257void ufshcd_remove(struct ufs_hba *hba) 5258{ 5259 scsi_remove_host(hba->host); 5260 /* disable interrupts */ 5261 ufshcd_disable_intr(hba, hba->intr_mask); 5262 ufshcd_hba_stop(hba); 5263 5264 scsi_host_put(hba->host); 5265 5266 ufshcd_exit_clk_gating(hba); 5267 if (ufshcd_is_clkscaling_enabled(hba)) 5268 devfreq_remove_device(hba->devfreq); 5269 ufshcd_hba_exit(hba); 5270} 5271EXPORT_SYMBOL_GPL(ufshcd_remove); 5272 5273/** 5274 * ufshcd_set_dma_mask - Set dma mask based on the controller 5275 * addressing capability 5276 * @hba: per adapter instance 5277 * 5278 * Returns 0 for success, non-zero for failure 5279 */ 5280static int ufshcd_set_dma_mask(struct ufs_hba *hba) 5281{ 5282 if (hba->capabilities & MASK_64_ADDRESSING_SUPPORT) { 5283 if (!dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(64))) 5284 return 0; 5285 } 5286 return dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(32)); 5287} 5288 5289/** 5290 * ufshcd_alloc_host - allocate Host Bus Adapter (HBA) 5291 * @dev: pointer to device handle 5292 * @hba_handle: driver private handle 5293 * Returns 0 on success, non-zero value on failure 5294 */ 5295int ufshcd_alloc_host(struct device *dev, struct ufs_hba **hba_handle) 5296{ 5297 struct Scsi_Host *host; 5298 struct ufs_hba *hba; 5299 int err = 0; 5300 5301 if (!dev) { 5302 dev_err(dev, 5303 "Invalid memory reference for dev is NULL\n"); 5304 err = -ENODEV; 5305 goto out_error; 5306 } 5307 5308 host = scsi_host_alloc(&ufshcd_driver_template, 5309 sizeof(struct ufs_hba)); 5310 if (!host) { 5311 dev_err(dev, "scsi_host_alloc failed\n"); 5312 err = -ENOMEM; 5313 goto out_error; 5314 } 5315 hba = shost_priv(host); 5316 hba->host = host; 5317 hba->dev = dev; 5318 *hba_handle = hba; 5319 5320out_error: 5321 return err; 5322} 5323EXPORT_SYMBOL(ufshcd_alloc_host); 5324 5325static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up) 5326{ 5327 int ret = 0; 5328 struct ufs_clk_info *clki; 5329 struct list_head *head = &hba->clk_list_head; 5330 5331 if (!head || list_empty(head)) 5332 goto out; 5333 5334 list_for_each_entry(clki, head, list) { 5335 if (!IS_ERR_OR_NULL(clki->clk)) { 5336 if (scale_up && clki->max_freq) { 5337 if (clki->curr_freq == clki->max_freq) 5338 continue; 5339 ret = clk_set_rate(clki->clk, clki->max_freq); 5340 if (ret) { 5341 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n", 5342 __func__, clki->name, 5343 clki->max_freq, ret); 5344 break; 5345 } 5346 clki->curr_freq = clki->max_freq; 5347 5348 } else if (!scale_up && clki->min_freq) { 5349 if (clki->curr_freq == clki->min_freq) 5350 continue; 5351 ret = clk_set_rate(clki->clk, clki->min_freq); 5352 if (ret) { 5353 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n", 5354 __func__, clki->name, 5355 clki->min_freq, ret); 5356 break; 5357 } 5358 clki->curr_freq = clki->min_freq; 5359 } 5360 } 5361 dev_dbg(hba->dev, "%s: clk: %s, rate: %lu\n", __func__, 5362 clki->name, clk_get_rate(clki->clk)); 5363 } 5364 if (hba->vops->clk_scale_notify) 5365 hba->vops->clk_scale_notify(hba); 5366out: 5367 return ret; 5368} 5369 5370static int ufshcd_devfreq_target(struct device *dev, 5371 unsigned long *freq, u32 flags) 5372{ 5373 int err = 0; 5374 struct ufs_hba *hba = dev_get_drvdata(dev); 5375 5376 if (!ufshcd_is_clkscaling_enabled(hba)) 5377 return -EINVAL; 5378 5379 if (*freq == UINT_MAX) 5380 err = ufshcd_scale_clks(hba, true); 5381 else if (*freq == 0) 5382 err = ufshcd_scale_clks(hba, false); 5383 5384 return err; 5385} 5386 5387static int ufshcd_devfreq_get_dev_status(struct device *dev, 5388 struct devfreq_dev_status *stat) 5389{ 5390 struct ufs_hba *hba = dev_get_drvdata(dev); 5391 struct ufs_clk_scaling *scaling = &hba->clk_scaling; 5392 unsigned long flags; 5393 5394 if (!ufshcd_is_clkscaling_enabled(hba)) 5395 return -EINVAL; 5396 5397 memset(stat, 0, sizeof(*stat)); 5398 5399 spin_lock_irqsave(hba->host->host_lock, flags); 5400 if (!scaling->window_start_t) 5401 goto start_window; 5402 5403 if (scaling->is_busy_started) 5404 scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(), 5405 scaling->busy_start_t)); 5406 5407 stat->total_time = jiffies_to_usecs((long)jiffies - 5408 (long)scaling->window_start_t); 5409 stat->busy_time = scaling->tot_busy_t; 5410start_window: 5411 scaling->window_start_t = jiffies; 5412 scaling->tot_busy_t = 0; 5413 5414 if (hba->outstanding_reqs) { 5415 scaling->busy_start_t = ktime_get(); 5416 scaling->is_busy_started = true; 5417 } else { 5418 scaling->busy_start_t = ktime_set(0, 0); 5419 scaling->is_busy_started = false; 5420 } 5421 spin_unlock_irqrestore(hba->host->host_lock, flags); 5422 return 0; 5423} 5424 5425static struct devfreq_dev_profile ufs_devfreq_profile = { 5426 .polling_ms = 100, 5427 .target = ufshcd_devfreq_target, 5428 .get_dev_status = ufshcd_devfreq_get_dev_status, 5429}; 5430 5431/** 5432 * ufshcd_init - Driver initialization routine 5433 * @hba: per-adapter instance 5434 * @mmio_base: base register address 5435 * @irq: Interrupt line of device 5436 * Returns 0 on success, non-zero value on failure 5437 */ 5438int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) 5439{ 5440 int err; 5441 struct Scsi_Host *host = hba->host; 5442 struct device *dev = hba->dev; 5443 5444 if (!mmio_base) { 5445 dev_err(hba->dev, 5446 "Invalid memory reference for mmio_base is NULL\n"); 5447 err = -ENODEV; 5448 goto out_error; 5449 } 5450 5451 hba->mmio_base = mmio_base; 5452 hba->irq = irq; 5453 5454 err = ufshcd_hba_init(hba); 5455 if (err) 5456 goto out_error; 5457 5458 /* Read capabilities registers */ 5459 ufshcd_hba_capabilities(hba); 5460 5461 /* Get UFS version supported by the controller */ 5462 hba->ufs_version = ufshcd_get_ufs_version(hba); 5463 5464 /* Get Interrupt bit mask per version */ 5465 hba->intr_mask = ufshcd_get_intr_mask(hba); 5466 5467 err = ufshcd_set_dma_mask(hba); 5468 if (err) { 5469 dev_err(hba->dev, "set dma mask failed\n"); 5470 goto out_disable; 5471 } 5472 5473 /* Allocate memory for host memory space */ 5474 err = ufshcd_memory_alloc(hba); 5475 if (err) { 5476 dev_err(hba->dev, "Memory allocation failed\n"); 5477 goto out_disable; 5478 } 5479 5480 /* Configure LRB */ 5481 ufshcd_host_memory_configure(hba); 5482 5483 host->can_queue = hba->nutrs; 5484 host->cmd_per_lun = hba->nutrs; 5485 host->max_id = UFSHCD_MAX_ID; 5486 host->max_lun = UFS_MAX_LUNS; 5487 host->max_channel = UFSHCD_MAX_CHANNEL; 5488 host->unique_id = host->host_no; 5489 host->max_cmd_len = MAX_CDB_SIZE; 5490 5491 hba->max_pwr_info.is_valid = false; 5492 5493 /* Initailize wait queue for task management */ 5494 init_waitqueue_head(&hba->tm_wq); 5495 init_waitqueue_head(&hba->tm_tag_wq); 5496 5497 /* Initialize work queues */ 5498 INIT_WORK(&hba->eh_work, ufshcd_err_handler); 5499 INIT_WORK(&hba->eeh_work, ufshcd_exception_event_handler); 5500 5501 /* Initialize UIC command mutex */ 5502 mutex_init(&hba->uic_cmd_mutex); 5503 5504 /* Initialize mutex for device management commands */ 5505 mutex_init(&hba->dev_cmd.lock); 5506 5507 /* Initialize device management tag acquire wait queue */ 5508 init_waitqueue_head(&hba->dev_cmd.tag_wq); 5509 5510 ufshcd_init_clk_gating(hba); 5511 /* IRQ registration */ 5512 err = devm_request_irq(dev, irq, ufshcd_intr, IRQF_SHARED, UFSHCD, hba); 5513 if (err) { 5514 dev_err(hba->dev, "request irq failed\n"); 5515 goto exit_gating; 5516 } else { 5517 hba->is_irq_enabled = true; 5518 } 5519 5520 /* Enable SCSI tag mapping */ 5521 err = scsi_init_shared_tag_map(host, host->can_queue); 5522 if (err) { 5523 dev_err(hba->dev, "init shared queue failed\n"); 5524 goto exit_gating; 5525 } 5526 5527 err = scsi_add_host(host, hba->dev); 5528 if (err) { 5529 dev_err(hba->dev, "scsi_add_host failed\n"); 5530 goto exit_gating; 5531 } 5532 5533 /* Host controller enable */ 5534 err = ufshcd_hba_enable(hba); 5535 if (err) { 5536 dev_err(hba->dev, "Host controller enable failed\n"); 5537 goto out_remove_scsi_host; 5538 } 5539 5540 if (ufshcd_is_clkscaling_enabled(hba)) { 5541 hba->devfreq = devfreq_add_device(dev, &ufs_devfreq_profile, 5542 "simple_ondemand", NULL); 5543 if (IS_ERR(hba->devfreq)) { 5544 dev_err(hba->dev, "Unable to register with devfreq %ld\n", 5545 PTR_ERR(hba->devfreq)); 5546 goto out_remove_scsi_host; 5547 } 5548 /* Suspend devfreq until the UFS device is detected */ 5549 devfreq_suspend_device(hba->devfreq); 5550 hba->clk_scaling.window_start_t = 0; 5551 } 5552 5553 /* Hold auto suspend until async scan completes */ 5554 pm_runtime_get_sync(dev); 5555 5556 /* 5557 * The device-initialize-sequence hasn't been invoked yet. 5558 * Set the device to power-off state 5559 */ 5560 ufshcd_set_ufs_dev_poweroff(hba); 5561 5562 async_schedule(ufshcd_async_scan, hba); 5563 5564 return 0; 5565 5566out_remove_scsi_host: 5567 scsi_remove_host(hba->host); 5568exit_gating: 5569 ufshcd_exit_clk_gating(hba); 5570out_disable: 5571 hba->is_irq_enabled = false; 5572 scsi_host_put(host); 5573 ufshcd_hba_exit(hba); 5574out_error: 5575 return err; 5576} 5577EXPORT_SYMBOL_GPL(ufshcd_init); 5578 5579MODULE_AUTHOR("Santosh Yaragnavi <santosh.sy@samsung.com>"); 5580MODULE_AUTHOR("Vinayak Holikatti <h.vinayak@samsung.com>"); 5581MODULE_DESCRIPTION("Generic UFS host controller driver Core"); 5582MODULE_LICENSE("GPL"); 5583MODULE_VERSION(UFSHCD_DRIVER_VERSION); 5584