diff options
Diffstat (limited to 'drivers/usb/dwc2/gadget.c')
| -rw-r--r-- | drivers/usb/dwc2/gadget.c | 1431 |
1 files changed, 1098 insertions, 333 deletions
diff --git a/drivers/usb/dwc2/gadget.c b/drivers/usb/dwc2/gadget.c index 24fbebc9b409..bc3b3fda5000 100644 --- a/drivers/usb/dwc2/gadget.c +++ b/drivers/usb/dwc2/gadget.c @@ -93,7 +93,18 @@ static void dwc2_hsotg_dump(struct dwc2_hsotg *hsotg); */ static inline bool using_dma(struct dwc2_hsotg *hsotg) { - return hsotg->g_using_dma; + return hsotg->params.g_dma; +} + +/* + * using_desc_dma - return the descriptor DMA status of the driver. + * @hsotg: The driver state. + * + * Return true if we're using descriptor DMA. + */ +static inline bool using_desc_dma(struct dwc2_hsotg *hsotg) +{ + return hsotg->params.g_dma_desc; } /** @@ -160,7 +171,7 @@ static void dwc2_hsotg_disable_gsint(struct dwc2_hsotg *hsotg, u32 ints) * request. */ static void dwc2_hsotg_ctrl_epint(struct dwc2_hsotg *hsotg, - unsigned int ep, unsigned int dir_in, + unsigned int ep, unsigned int dir_in, unsigned int en) { unsigned long flags; @@ -181,6 +192,99 @@ static void dwc2_hsotg_ctrl_epint(struct dwc2_hsotg *hsotg, } /** + * dwc2_hsotg_tx_fifo_count - return count of TX FIFOs in device mode + */ +int dwc2_hsotg_tx_fifo_count(struct dwc2_hsotg *hsotg) +{ + if (hsotg->hw_params.en_multiple_tx_fifo) + /* In dedicated FIFO mode we need count of IN EPs */ + return (dwc2_readl(hsotg->regs + GHWCFG4) & + GHWCFG4_NUM_IN_EPS_MASK) >> GHWCFG4_NUM_IN_EPS_SHIFT; + else + /* In shared FIFO mode we need count of Periodic IN EPs */ + return hsotg->hw_params.num_dev_perio_in_ep; +} + +/** + * dwc2_hsotg_ep_info_size - return Endpoint Info Control block size in DWORDs + */ +static int dwc2_hsotg_ep_info_size(struct dwc2_hsotg *hsotg) +{ + int val = 0; + int i; + u32 ep_dirs; + + /* + * Don't need additional space for ep info control registers in + * slave mode. + */ + if (!using_dma(hsotg)) { + dev_dbg(hsotg->dev, "Buffer DMA ep info size 0\n"); + return 0; + } + + /* + * Buffer DMA mode - 1 location per endpoit + * Descriptor DMA mode - 4 locations per endpoint + */ + ep_dirs = hsotg->hw_params.dev_ep_dirs; + + for (i = 0; i <= hsotg->hw_params.num_dev_ep; i++) { + val += ep_dirs & 3 ? 1 : 2; + ep_dirs >>= 2; + } + + if (using_desc_dma(hsotg)) + val = val * 4; + + return val; +} + +/** + * dwc2_hsotg_tx_fifo_total_depth - return total FIFO depth available for + * device mode TX FIFOs + */ +int dwc2_hsotg_tx_fifo_total_depth(struct dwc2_hsotg *hsotg) +{ + int ep_info_size; + int addr; + int tx_addr_max; + u32 np_tx_fifo_size; + + np_tx_fifo_size = min_t(u32, hsotg->hw_params.dev_nperio_tx_fifo_size, + hsotg->params.g_np_tx_fifo_size); + + /* Get Endpoint Info Control block size in DWORDs. */ + ep_info_size = dwc2_hsotg_ep_info_size(hsotg); + tx_addr_max = hsotg->hw_params.total_fifo_size - ep_info_size; + + addr = hsotg->params.g_rx_fifo_size + np_tx_fifo_size; + if (tx_addr_max <= addr) + return 0; + + return tx_addr_max - addr; +} + +/** + * dwc2_hsotg_tx_fifo_average_depth - returns average depth of device mode + * TX FIFOs + */ +int dwc2_hsotg_tx_fifo_average_depth(struct dwc2_hsotg *hsotg) +{ + int tx_fifo_count; + int tx_fifo_depth; + + tx_fifo_depth = dwc2_hsotg_tx_fifo_total_depth(hsotg); + + tx_fifo_count = dwc2_hsotg_tx_fifo_count(hsotg); + + if (!tx_fifo_count) + return tx_fifo_depth; + else + return tx_fifo_depth / tx_fifo_count; +} + +/** * dwc2_hsotg_init_fifo - initialise non-periodic FIFOs * @hsotg: The device instance. */ @@ -190,16 +294,17 @@ static void dwc2_hsotg_init_fifo(struct dwc2_hsotg *hsotg) unsigned int addr; int timeout; u32 val; + u32 *txfsz = hsotg->params.g_tx_fifo_size; /* Reset fifo map if not correctly cleared during previous session */ WARN_ON(hsotg->fifo_map); hsotg->fifo_map = 0; /* set RX/NPTX FIFO sizes */ - dwc2_writel(hsotg->g_rx_fifo_sz, hsotg->regs + GRXFSIZ); - dwc2_writel((hsotg->g_rx_fifo_sz << FIFOSIZE_STARTADDR_SHIFT) | - (hsotg->g_np_g_tx_fifo_sz << FIFOSIZE_DEPTH_SHIFT), - hsotg->regs + GNPTXFSIZ); + dwc2_writel(hsotg->params.g_rx_fifo_size, hsotg->regs + GRXFSIZ); + dwc2_writel((hsotg->params.g_rx_fifo_size << FIFOSIZE_STARTADDR_SHIFT) | + (hsotg->params.g_np_tx_fifo_size << FIFOSIZE_DEPTH_SHIFT), + hsotg->regs + GNPTXFSIZ); /* * arange all the rest of the TX FIFOs, as some versions of this @@ -209,7 +314,7 @@ static void dwc2_hsotg_init_fifo(struct dwc2_hsotg *hsotg) */ /* start at the end of the GNPTXFSIZ, rounded up */ - addr = hsotg->g_rx_fifo_sz + hsotg->g_np_g_tx_fifo_sz; + addr = hsotg->params.g_rx_fifo_size + hsotg->params.g_np_tx_fifo_size; /* * Configure fifos sizes from provided configuration and assign @@ -217,17 +322,21 @@ static void dwc2_hsotg_init_fifo(struct dwc2_hsotg *hsotg) * given endpoint. */ for (ep = 1; ep < MAX_EPS_CHANNELS; ep++) { - if (!hsotg->g_tx_fifo_sz[ep]) + if (!txfsz[ep]) continue; val = addr; - val |= hsotg->g_tx_fifo_sz[ep] << FIFOSIZE_DEPTH_SHIFT; - WARN_ONCE(addr + hsotg->g_tx_fifo_sz[ep] > hsotg->fifo_mem, + val |= txfsz[ep] << FIFOSIZE_DEPTH_SHIFT; + WARN_ONCE(addr + txfsz[ep] > hsotg->fifo_mem, "insufficient fifo memory"); - addr += hsotg->g_tx_fifo_sz[ep]; + addr += txfsz[ep]; dwc2_writel(val, hsotg->regs + DPTXFSIZN(ep)); + val = dwc2_readl(hsotg->regs + DPTXFSIZN(ep)); } + dwc2_writel(hsotg->hw_params.total_fifo_size | + addr << GDFIFOCFG_EPINFOBASE_SHIFT, + hsotg->regs + GDFIFOCFG); /* * according to p428 of the design guide, we need to ensure that * all fifos are flushed before continuing @@ -264,11 +373,11 @@ static void dwc2_hsotg_init_fifo(struct dwc2_hsotg *hsotg) * Allocate a new USB request structure appropriate for the specified endpoint */ static struct usb_request *dwc2_hsotg_ep_alloc_request(struct usb_ep *ep, - gfp_t flags) + gfp_t flags) { struct dwc2_hsotg_req *req; - req = kzalloc(sizeof(struct dwc2_hsotg_req), flags); + req = kzalloc(sizeof(*req), flags); if (!req) return NULL; @@ -299,18 +408,62 @@ static inline int is_ep_periodic(struct dwc2_hsotg_ep *hs_ep) * of a request to ensure the buffer is ready for access by the caller. */ static void dwc2_hsotg_unmap_dma(struct dwc2_hsotg *hsotg, - struct dwc2_hsotg_ep *hs_ep, + struct dwc2_hsotg_ep *hs_ep, struct dwc2_hsotg_req *hs_req) { struct usb_request *req = &hs_req->req; - /* ignore this if we're not moving any data */ - if (hs_req->req.length == 0) - return; - usb_gadget_unmap_request(&hsotg->gadget, req, hs_ep->dir_in); } +/* + * dwc2_gadget_alloc_ctrl_desc_chains - allocate DMA descriptor chains + * for Control endpoint + * @hsotg: The device state. + * + * This function will allocate 4 descriptor chains for EP 0: 2 for + * Setup stage, per one for IN and OUT data/status transactions. + */ +static int dwc2_gadget_alloc_ctrl_desc_chains(struct dwc2_hsotg *hsotg) +{ + hsotg->setup_desc[0] = + dmam_alloc_coherent(hsotg->dev, + sizeof(struct dwc2_dma_desc), + &hsotg->setup_desc_dma[0], + GFP_KERNEL); + if (!hsotg->setup_desc[0]) + goto fail; + + hsotg->setup_desc[1] = + dmam_alloc_coherent(hsotg->dev, + sizeof(struct dwc2_dma_desc), + &hsotg->setup_desc_dma[1], + GFP_KERNEL); + if (!hsotg->setup_desc[1]) + goto fail; + + hsotg->ctrl_in_desc = + dmam_alloc_coherent(hsotg->dev, + sizeof(struct dwc2_dma_desc), + &hsotg->ctrl_in_desc_dma, + GFP_KERNEL); + if (!hsotg->ctrl_in_desc) + goto fail; + + hsotg->ctrl_out_desc = + dmam_alloc_coherent(hsotg->dev, + sizeof(struct dwc2_dma_desc), + &hsotg->ctrl_out_desc_dma, + GFP_KERNEL); + if (!hsotg->ctrl_out_desc) + goto fail; + + return 0; + +fail: + return -ENOMEM; +} + /** * dwc2_hsotg_write_fifo - write packet Data to the TxFIFO * @hsotg: The controller state. @@ -328,7 +481,7 @@ static void dwc2_hsotg_unmap_dma(struct dwc2_hsotg *hsotg, * This routine is only needed for PIO */ static int dwc2_hsotg_write_fifo(struct dwc2_hsotg *hsotg, - struct dwc2_hsotg_ep *hs_ep, + struct dwc2_hsotg_ep *hs_ep, struct dwc2_hsotg_req *hs_req) { bool periodic = is_ep_periodic(hs_ep); @@ -410,7 +563,7 @@ static int dwc2_hsotg_write_fifo(struct dwc2_hsotg *hsotg, max_transfer = hs_ep->ep.maxpacket * hs_ep->mc; dev_dbg(hsotg->dev, "%s: GNPTXSTS=%08x, can=%d, to=%d, max_transfer %d\n", - __func__, gnptxsts, can_write, to_write, max_transfer); + __func__, gnptxsts, can_write, to_write, max_transfer); /* * limit to 512 bytes of data, it seems at least on the non-periodic @@ -431,7 +584,7 @@ static int dwc2_hsotg_write_fifo(struct dwc2_hsotg *hsotg, /* it's needed only when we do not use dedicated fifos */ if (!hsotg->dedicated_fifos) dwc2_hsotg_en_gsint(hsotg, - periodic ? GINTSTS_PTXFEMP : + periodic ? GINTSTS_PTXFEMP : GINTSTS_NPTXFEMP); } @@ -460,12 +613,12 @@ static int dwc2_hsotg_write_fifo(struct dwc2_hsotg *hsotg, /* it's needed only when we do not use dedicated fifos */ if (!hsotg->dedicated_fifos) dwc2_hsotg_en_gsint(hsotg, - periodic ? GINTSTS_PTXFEMP : + periodic ? GINTSTS_PTXFEMP : GINTSTS_NPTXFEMP); } dev_dbg(hsotg->dev, "write %d/%d, can_write %d, done %d\n", - to_write, hs_req->req.length, can_write, buf_pos); + to_write, hs_req->req.length, can_write, buf_pos); if (to_write <= 0) return -ENOSPC; @@ -491,17 +644,17 @@ static int dwc2_hsotg_write_fifo(struct dwc2_hsotg *hsotg, * Return the maximum data that can be queued in one go on a given endpoint * so that transfers that are too long can be split. */ -static unsigned get_ep_limit(struct dwc2_hsotg_ep *hs_ep) +static unsigned int get_ep_limit(struct dwc2_hsotg_ep *hs_ep) { int index = hs_ep->index; - unsigned maxsize; - unsigned maxpkt; + unsigned int maxsize; + unsigned int maxpkt; if (index != 0) { maxsize = DXEPTSIZ_XFERSIZE_LIMIT + 1; maxpkt = DXEPTSIZ_PKTCNT_LIMIT + 1; } else { - maxsize = 64+64; + maxsize = 64 + 64; if (hs_ep->dir_in) maxpkt = DIEPTSIZ0_PKTCNT_LIMIT + 1; else @@ -524,11 +677,11 @@ static unsigned get_ep_limit(struct dwc2_hsotg_ep *hs_ep) } /** -* dwc2_hsotg_read_frameno - read current frame number -* @hsotg: The device instance -* -* Return the current frame number -*/ + * dwc2_hsotg_read_frameno - read current frame number + * @hsotg: The device instance + * + * Return the current frame number + */ static u32 dwc2_hsotg_read_frameno(struct dwc2_hsotg *hsotg) { u32 dsts; @@ -541,6 +694,273 @@ static u32 dwc2_hsotg_read_frameno(struct dwc2_hsotg *hsotg) } /** + * dwc2_gadget_get_chain_limit - get the maximum data payload value of the + * DMA descriptor chain prepared for specific endpoint + * @hs_ep: The endpoint + * + * Return the maximum data that can be queued in one go on a given endpoint + * depending on its descriptor chain capacity so that transfers that + * are too long can be split. + */ +static unsigned int dwc2_gadget_get_chain_limit(struct dwc2_hsotg_ep *hs_ep) +{ + int is_isoc = hs_ep->isochronous; + unsigned int maxsize; + + if (is_isoc) + maxsize = hs_ep->dir_in ? DEV_DMA_ISOC_TX_NBYTES_LIMIT : + DEV_DMA_ISOC_RX_NBYTES_LIMIT; + else + maxsize = DEV_DMA_NBYTES_LIMIT; + + /* Above size of one descriptor was chosen, multiple it */ + maxsize *= MAX_DMA_DESC_NUM_GENERIC; + + return maxsize; +} + +/* + * dwc2_gadget_get_desc_params - get DMA descriptor parameters. + * @hs_ep: The endpoint + * @mask: RX/TX bytes mask to be defined + * + * Returns maximum data payload for one descriptor after analyzing endpoint + * characteristics. + * DMA descriptor transfer bytes limit depends on EP type: + * Control out - MPS, + * Isochronous - descriptor rx/tx bytes bitfield limit, + * Control In/Bulk/Interrupt - multiple of mps. This will allow to not + * have concatenations from various descriptors within one packet. + * + * Selects corresponding mask for RX/TX bytes as well. + */ +static u32 dwc2_gadget_get_desc_params(struct dwc2_hsotg_ep *hs_ep, u32 *mask) +{ + u32 mps = hs_ep->ep.maxpacket; + int dir_in = hs_ep->dir_in; + u32 desc_size = 0; + + if (!hs_ep->index && !dir_in) { + desc_size = mps; + *mask = DEV_DMA_NBYTES_MASK; + } else if (hs_ep->isochronous) { + if (dir_in) { + desc_size = DEV_DMA_ISOC_TX_NBYTES_LIMIT; + *mask = DEV_DMA_ISOC_TX_NBYTES_MASK; + } else { + desc_size = DEV_DMA_ISOC_RX_NBYTES_LIMIT; + *mask = DEV_DMA_ISOC_RX_NBYTES_MASK; + } + } else { + desc_size = DEV_DMA_NBYTES_LIMIT; + *mask = DEV_DMA_NBYTES_MASK; + + /* Round down desc_size to be mps multiple */ + desc_size -= desc_size % mps; + } + + return desc_size; +} + +/* + * dwc2_gadget_config_nonisoc_xfer_ddma - prepare non ISOC DMA desc chain. + * @hs_ep: The endpoint + * @dma_buff: DMA address to use + * @len: Length of the transfer + * + * This function will iterate over descriptor chain and fill its entries + * with corresponding information based on transfer data. + */ +static void dwc2_gadget_config_nonisoc_xfer_ddma(struct dwc2_hsotg_ep *hs_ep, + dma_addr_t dma_buff, + unsigned int len) +{ + struct dwc2_hsotg *hsotg = hs_ep->parent; + int dir_in = hs_ep->dir_in; + struct dwc2_dma_desc *desc = hs_ep->desc_list; + u32 mps = hs_ep->ep.maxpacket; + u32 maxsize = 0; + u32 offset = 0; + u32 mask = 0; + int i; + + maxsize = dwc2_gadget_get_desc_params(hs_ep, &mask); + + hs_ep->desc_count = (len / maxsize) + + ((len % maxsize) ? 1 : 0); + if (len == 0) + hs_ep->desc_count = 1; + + for (i = 0; i < hs_ep->desc_count; ++i) { + desc->status = 0; + desc->status |= (DEV_DMA_BUFF_STS_HBUSY + << DEV_DMA_BUFF_STS_SHIFT); + + if (len > maxsize) { + if (!hs_ep->index && !dir_in) + desc->status |= (DEV_DMA_L | DEV_DMA_IOC); + + desc->status |= (maxsize << + DEV_DMA_NBYTES_SHIFT & mask); + desc->buf = dma_buff + offset; + + len -= maxsize; + offset += maxsize; + } else { + desc->status |= (DEV_DMA_L | DEV_DMA_IOC); + + if (dir_in) + desc->status |= (len % mps) ? DEV_DMA_SHORT : + ((hs_ep->send_zlp) ? DEV_DMA_SHORT : 0); + if (len > maxsize) + dev_err(hsotg->dev, "wrong len %d\n", len); + + desc->status |= + len << DEV_DMA_NBYTES_SHIFT & mask; + desc->buf = dma_buff + offset; + } + + desc->status &= ~DEV_DMA_BUFF_STS_MASK; + desc->status |= (DEV_DMA_BUFF_STS_HREADY + << DEV_DMA_BUFF_STS_SHIFT); + desc++; + } +} + +/* + * dwc2_gadget_fill_isoc_desc - fills next isochronous descriptor in chain. + * @hs_ep: The isochronous endpoint. + * @dma_buff: usb requests dma buffer. + * @len: usb request transfer length. + * + * Finds out index of first free entry either in the bottom or up half of + * descriptor chain depend on which is under SW control and not processed + * by HW. Then fills that descriptor with the data of the arrived usb request, + * frame info, sets Last and IOC bits increments next_desc. If filled + * descriptor is not the first one, removes L bit from the previous descriptor + * status. + */ +static int dwc2_gadget_fill_isoc_desc(struct dwc2_hsotg_ep *hs_ep, + dma_addr_t dma_buff, unsigned int len) +{ + struct dwc2_dma_desc *desc; + struct dwc2_hsotg *hsotg = hs_ep->parent; + u32 index; + u32 maxsize = 0; + u32 mask = 0; + + maxsize = dwc2_gadget_get_desc_params(hs_ep, &mask); + if (len > maxsize) { + dev_err(hsotg->dev, "wrong len %d\n", len); + return -EINVAL; + } + + /* + * If SW has already filled half of chain, then return and wait for + * the other chain to be processed by HW. + */ + if (hs_ep->next_desc == MAX_DMA_DESC_NUM_GENERIC / 2) + return -EBUSY; + + /* Increment frame number by interval for IN */ + if (hs_ep->dir_in) + dwc2_gadget_incr_frame_num(hs_ep); + + index = (MAX_DMA_DESC_NUM_GENERIC / 2) * hs_ep->isoc_chain_num + + hs_ep->next_desc; + + /* Sanity check of calculated index */ + if ((hs_ep->isoc_chain_num && index > MAX_DMA_DESC_NUM_GENERIC) || + (!hs_ep->isoc_chain_num && index > MAX_DMA_DESC_NUM_GENERIC / 2)) { + dev_err(hsotg->dev, "wrong index %d for iso chain\n", index); + return -EINVAL; + } + + desc = &hs_ep->desc_list[index]; + + /* Clear L bit of previous desc if more than one entries in the chain */ + if (hs_ep->next_desc) + hs_ep->desc_list[index - 1].status &= ~DEV_DMA_L; + + dev_dbg(hsotg->dev, "%s: Filling ep %d, dir %s isoc desc # %d\n", + __func__, hs_ep->index, hs_ep->dir_in ? "in" : "out", index); + + desc->status = 0; + desc->status |= (DEV_DMA_BUFF_STS_HBUSY << DEV_DMA_BUFF_STS_SHIFT); + + desc->buf = dma_buff; + desc->status |= (DEV_DMA_L | DEV_DMA_IOC | + ((len << DEV_DMA_NBYTES_SHIFT) & mask)); + + if (hs_ep->dir_in) { + desc->status |= ((hs_ep->mc << DEV_DMA_ISOC_PID_SHIFT) & + DEV_DMA_ISOC_PID_MASK) | + ((len % hs_ep->ep.maxpacket) ? + DEV_DMA_SHORT : 0) | + ((hs_ep->target_frame << + DEV_DMA_ISOC_FRNUM_SHIFT) & + DEV_DMA_ISOC_FRNUM_MASK); + } + + desc->status &= ~DEV_DMA_BUFF_STS_MASK; + desc->status |= (DEV_DMA_BUFF_STS_HREADY << DEV_DMA_BUFF_STS_SHIFT); + + /* Update index of last configured entry in the chain */ + hs_ep->next_desc++; + + return 0; +} + +/* + * dwc2_gadget_start_isoc_ddma - start isochronous transfer in DDMA + * @hs_ep: The isochronous endpoint. + * + * Prepare first descriptor chain for isochronous endpoints. Afterwards + * write DMA address to HW and enable the endpoint. + * + * Switch between descriptor chains via isoc_chain_num to give SW opportunity + * to prepare second descriptor chain while first one is being processed by HW. + */ +static void dwc2_gadget_start_isoc_ddma(struct dwc2_hsotg_ep *hs_ep) +{ + struct dwc2_hsotg *hsotg = hs_ep->parent; + struct dwc2_hsotg_req *hs_req, *treq; + int index = hs_ep->index; + int ret; + u32 dma_reg; + u32 depctl; + u32 ctrl; + + if (list_empty(&hs_ep->queue)) { + dev_dbg(hsotg->dev, "%s: No requests in queue\n", __func__); + return; + } + + list_for_each_entry_safe(hs_req, treq, &hs_ep->queue, queue) { + ret = dwc2_gadget_fill_isoc_desc(hs_ep, hs_req->req.dma, + hs_req->req.length); + if (ret) { + dev_dbg(hsotg->dev, "%s: desc chain full\n", __func__); + break; + } + } + + depctl = hs_ep->dir_in ? DIEPCTL(index) : DOEPCTL(index); + dma_reg = hs_ep->dir_in ? DIEPDMA(index) : DOEPDMA(index); + + /* write descriptor chain address to control register */ + dwc2_writel(hs_ep->desc_list_dma, hsotg->regs + dma_reg); + + ctrl = dwc2_readl(hsotg->regs + depctl); + ctrl |= DXEPCTL_EPENA | DXEPCTL_CNAK; + dwc2_writel(ctrl, hsotg->regs + depctl); + + /* Switch ISOC descriptor chain number being processed by SW*/ + hs_ep->isoc_chain_num = (hs_ep->isoc_chain_num ^ 1) & 0x1; + hs_ep->next_desc = 0; +} + +/** * dwc2_hsotg_start_req - start a USB request from an endpoint's queue * @hsotg: The controller state. * @hs_ep: The endpoint to process a request for @@ -551,7 +971,7 @@ static u32 dwc2_hsotg_read_frameno(struct dwc2_hsotg *hsotg) * appropriately, and writing any data to the FIFOs. */ static void dwc2_hsotg_start_req(struct dwc2_hsotg *hsotg, - struct dwc2_hsotg_ep *hs_ep, + struct dwc2_hsotg_ep *hs_ep, struct dwc2_hsotg_req *hs_req, bool continuing) { @@ -562,9 +982,10 @@ static void dwc2_hsotg_start_req(struct dwc2_hsotg *hsotg, u32 epsize_reg; u32 epsize; u32 ctrl; - unsigned length; - unsigned packets; - unsigned maxreq; + unsigned int length; + unsigned int packets; + unsigned int maxreq; + unsigned int dma_reg; if (index != 0) { if (hs_ep->req && !continuing) { @@ -579,6 +1000,7 @@ static void dwc2_hsotg_start_req(struct dwc2_hsotg *hsotg, } } + dma_reg = dir_in ? DIEPDMA(index) : DOEPDMA(index); epctrl_reg = dir_in ? DIEPCTL(index) : DOEPCTL(index); epsize_reg = dir_in ? DIEPTSIZ(index) : DOEPTSIZ(index); @@ -598,7 +1020,11 @@ static void dwc2_hsotg_start_req(struct dwc2_hsotg *hsotg, dev_dbg(hsotg->dev, "ureq->length:%d ureq->actual:%d\n", ureq->length, ureq->actual); - maxreq = get_ep_limit(hs_ep); + if (!using_desc_dma(hsotg)) + maxreq = get_ep_limit(hs_ep); + else + maxreq = dwc2_gadget_get_chain_limit(hs_ep); + if (length > maxreq) { int round = maxreq % hs_ep->ep.maxpacket; @@ -637,7 +1063,7 @@ static void dwc2_hsotg_start_req(struct dwc2_hsotg *hsotg, if (dir_in && ureq->zero && !continuing) { /* Test if zlp is actually required. */ if ((ureq->length >= hs_ep->ep.maxpacket) && - !(ureq->length % hs_ep->ep.maxpacket)) + !(ureq->length % hs_ep->ep.maxpacket)) hs_ep->send_zlp = 1; } @@ -650,22 +1076,51 @@ static void dwc2_hsotg_start_req(struct dwc2_hsotg *hsotg, /* store the request as the current one we're doing */ hs_ep->req = hs_req; - /* write size / packets */ - dwc2_writel(epsize, hsotg->regs + epsize_reg); + if (using_desc_dma(hsotg)) { + u32 offset = 0; + u32 mps = hs_ep->ep.maxpacket; - if (using_dma(hsotg) && !continuing) { - unsigned int dma_reg; + /* Adjust length: EP0 - MPS, other OUT EPs - multiple of MPS */ + if (!dir_in) { + if (!index) + length = mps; + else if (length % mps) + length += (mps - (length % mps)); + } /* - * write DMA address to control register, buffer already - * synced by dwc2_hsotg_ep_queue(). + * If more data to send, adjust DMA for EP0 out data stage. + * ureq->dma stays unchanged, hence increment it by already + * passed passed data count before starting new transaction. */ + if (!index && hsotg->ep0_state == DWC2_EP0_DATA_OUT && + continuing) + offset = ureq->actual; + + /* Fill DDMA chain entries */ + dwc2_gadget_config_nonisoc_xfer_ddma(hs_ep, ureq->dma + offset, + length); - dma_reg = dir_in ? DIEPDMA(index) : DOEPDMA(index); - dwc2_writel(ureq->dma, hsotg->regs + dma_reg); + /* write descriptor chain address to control register */ + dwc2_writel(hs_ep->desc_list_dma, hsotg->regs + dma_reg); - dev_dbg(hsotg->dev, "%s: %pad => 0x%08x\n", - __func__, &ureq->dma, dma_reg); + dev_dbg(hsotg->dev, "%s: %08x pad => 0x%08x\n", + __func__, (u32)hs_ep->desc_list_dma, dma_reg); + } else { + /* write size / packets */ + dwc2_writel(epsize, hsotg->regs + epsize_reg); + + if (using_dma(hsotg) && !continuing && (length != 0)) { + /* + * write DMA address to control register, buffer + * already synced by dwc2_hsotg_ep_queue(). + */ + + dwc2_writel(ureq->dma, hsotg->regs + dma_reg); + + dev_dbg(hsotg->dev, "%s: %pad => 0x%08x\n", + __func__, &ureq->dma, dma_reg); + } } if (hs_ep->isochronous && hs_ep->interval == 1) { @@ -712,7 +1167,7 @@ static void dwc2_hsotg_start_req(struct dwc2_hsotg *hsotg, /* check ep is enabled */ if (!(dwc2_readl(hsotg->regs + epctrl_reg) & DXEPCTL_EPENA)) dev_dbg(hsotg->dev, - "ep%d: failed to become enabled (DXEPCTL=0x%08x)?\n", + "ep%d: failed to become enabled (DXEPCTL=0x%08x)?\n", index, dwc2_readl(hsotg->regs + epctrl_reg)); dev_dbg(hsotg->dev, "%s: DXEPCTL=0x%08x\n", @@ -735,16 +1190,11 @@ static void dwc2_hsotg_start_req(struct dwc2_hsotg *hsotg, * cleanup on completion. */ static int dwc2_hsotg_map_dma(struct dwc2_hsotg *hsotg, - struct dwc2_hsotg_ep *hs_ep, + struct dwc2_hsotg_ep *hs_ep, struct usb_request *req) { - struct dwc2_hsotg_req *hs_req = our_req(req); int ret; - /* if the length is zero, ignore the DMA data */ - if (hs_req->req.length == 0) - return 0; - ret = usb_gadget_map_request(&hsotg->gadget, req, hs_ep->dir_in); if (ret) goto dma_error; @@ -759,7 +1209,8 @@ dma_error: } static int dwc2_hsotg_handle_unaligned_buf_start(struct dwc2_hsotg *hsotg, - struct dwc2_hsotg_ep *hs_ep, struct dwc2_hsotg_req *hs_req) + struct dwc2_hsotg_ep *hs_ep, + struct dwc2_hsotg_req *hs_req) { void *req_buf = hs_req->req.buf; @@ -770,7 +1221,7 @@ static int dwc2_hsotg_handle_unaligned_buf_start(struct dwc2_hsotg *hsotg, WARN_ON(hs_req->saved_req_buf); dev_dbg(hsotg->dev, "%s: %s: buf=%p length=%d\n", __func__, - hs_ep->ep.name, req_buf, hs_req->req.length); + hs_ep->ep.name, req_buf, hs_req->req.length); hs_req->req.buf = kmalloc(hs_req->req.length, GFP_ATOMIC); if (!hs_req->req.buf) { @@ -789,8 +1240,10 @@ static int dwc2_hsotg_handle_unaligned_buf_start(struct dwc2_hsotg *hsotg, return 0; } -static void dwc2_hsotg_handle_unaligned_buf_complete(struct dwc2_hsotg *hsotg, - struct dwc2_hsotg_ep *hs_ep, struct dwc2_hsotg_req *hs_req) +static void +dwc2_hsotg_handle_unaligned_buf_complete(struct dwc2_hsotg *hsotg, + struct dwc2_hsotg_ep *hs_ep, + struct dwc2_hsotg_req *hs_req) { /* If dma is not being used or buffer was aligned */ if (!using_dma(hsotg) || !hs_req->saved_req_buf) @@ -802,7 +1255,7 @@ static void dwc2_hsotg_handle_unaligned_buf_complete(struct dwc2_hsotg *hsotg, /* Copy data from bounce buffer on successful out transfer */ if (!hs_ep->dir_in && !hs_req->req.status) memcpy(hs_req->saved_req_buf, hs_req->req.buf, - hs_req->req.actual); + hs_req->req.actual); /* Free bounce buffer */ kfree(hs_req->req.buf); @@ -835,8 +1288,43 @@ static bool dwc2_gadget_target_frame_elapsed(struct dwc2_hsotg_ep *hs_ep) return false; } +/* + * dwc2_gadget_set_ep0_desc_chain - Set EP's desc chain pointers + * @hsotg: The driver state + * @hs_ep: the ep descriptor chain is for + * + * Called to update EP0 structure's pointers depend on stage of + * control transfer. + */ +static int dwc2_gadget_set_ep0_desc_chain(struct dwc2_hsotg *hsotg, + struct dwc2_hsotg_ep *hs_ep) +{ + switch (hsotg->ep0_state) { + case DWC2_EP0_SETUP: + case DWC2_EP0_STATUS_OUT: + hs_ep->desc_list = hsotg->setup_desc[0]; + hs_ep->desc_list_dma = hsotg->setup_desc_dma[0]; + break; + case DWC2_EP0_DATA_IN: + case DWC2_EP0_STATUS_IN: + hs_ep->desc_list = hsotg->ctrl_in_desc; + hs_ep->desc_list_dma = hsotg->ctrl_in_desc_dma; + break; + case DWC2_EP0_DATA_OUT: + hs_ep->desc_list = hsotg->ctrl_out_desc; + hs_ep->desc_list_dma = hsotg->ctrl_out_desc_dma; + break; + default: + dev_err(hsotg->dev, "invalid EP 0 state in queue %d\n", + hsotg->ep0_state); + return -EINVAL; + } + + return 0; +} + static int dwc2_hsotg_ep_queue(struct usb_ep *ep, struct usb_request *req, - gfp_t gfp_flags) + gfp_t gfp_flags) { struct dwc2_hsotg_req *hs_req = our_req(req); struct dwc2_hsotg_ep *hs_ep = our_ep(ep); @@ -851,7 +1339,7 @@ static int dwc2_hsotg_ep_queue(struct usb_ep *ep, struct usb_request *req, /* Prevent new request submission when controller is suspended */ if (hs->lx_state == DWC2_L2) { dev_dbg(hs->dev, "%s: don't submit request while suspended\n", - __func__); + __func__); return -EAGAIN; } @@ -870,10 +1358,32 @@ static int dwc2_hsotg_ep_queue(struct usb_ep *ep, struct usb_request *req, if (ret) return ret; } + /* If using descriptor DMA configure EP0 descriptor chain pointers */ + if (using_desc_dma(hs) && !hs_ep->index) { + ret = dwc2_gadget_set_ep0_desc_chain(hs, hs_ep); + if (ret) + return ret; + } first = list_empty(&hs_ep->queue); list_add_tail(&hs_req->queue, &hs_ep->queue); + /* + * Handle DDMA isochronous transfers separately - just add new entry + * to the half of descriptor chain that is not processed by HW. + * Transfer will be started once SW gets either one of NAK or + * OutTknEpDis interrupts. + */ + if (using_desc_dma(hs) && hs_ep->isochronous && + hs_ep->target_frame != TARGET_FRAME_INITIAL) { + ret = dwc2_gadget_fill_isoc_desc(hs_ep, hs_req->req.dma, + hs_req->req.length); + if (ret) + dev_dbg(hs->dev, "%s: ISO desc chain full\n", __func__); + + return 0; + } + if (first) { if (!hs_ep->isochronous) { dwc2_hsotg_start_req(hs, hs_ep, hs_req, false); @@ -890,7 +1400,7 @@ static int dwc2_hsotg_ep_queue(struct usb_ep *ep, struct usb_request *req, } static int dwc2_hsotg_ep_queue_lock(struct usb_ep *ep, struct usb_request *req, - gfp_t gfp_flags) + gfp_t gfp_flags) { struct dwc2_hsotg_ep *hs_ep = our_ep(ep); struct dwc2_hsotg *hs = hs_ep->parent; @@ -905,7 +1415,7 @@ static int dwc2_hsotg_ep_queue_lock(struct usb_ep *ep, struct usb_request *req, } static void dwc2_hsotg_ep_free_request(struct usb_ep *ep, - struct usb_request *req) + struct usb_request *req) { struct dwc2_hsotg_req *hs_req = our_req(req); @@ -921,7 +1431,7 @@ static void dwc2_hsotg_ep_free_request(struct usb_ep *ep, * submitted that need cleaning up. */ static void dwc2_hsotg_complete_oursetup(struct usb_ep *ep, - struct usb_request *req) + struct usb_request *req) { struct dwc2_hsotg_ep *hs_ep = our_ep(ep); struct dwc2_hsotg *hsotg = hs_ep->parent; @@ -940,7 +1450,7 @@ static void dwc2_hsotg_complete_oursetup(struct usb_ep *ep, * structure, or return NULL if it is not a valid endpoint. */ static struct dwc2_hsotg_ep *ep_from_windex(struct dwc2_hsotg *hsotg, - u32 windex) + u32 windex) { struct dwc2_hsotg_ep *ep; int dir = (windex & USB_DIR_IN) ? 1 : 0; @@ -997,7 +1507,7 @@ int dwc2_hsotg_set_test_mode(struct dwc2_hsotg *hsotg, int testmode) * an internal method of sending replies to certain control requests, etc. */ static int dwc2_hsotg_send_reply(struct dwc2_hsotg *hsotg, - struct dwc2_hsotg_ep *ep, + struct dwc2_hsotg_ep *ep, void *buff, int length) { @@ -1040,7 +1550,7 @@ static int dwc2_hsotg_send_reply(struct dwc2_hsotg *hsotg, * @ctrl: USB control request */ static int dwc2_hsotg_process_req_status(struct dwc2_hsotg *hsotg, - struct usb_ctrlrequest *ctrl) + struct usb_ctrlrequest *ctrl) { struct dwc2_hsotg_ep *ep0 = hsotg->eps_out[0]; struct dwc2_hsotg_ep *ep; @@ -1056,8 +1566,11 @@ static int dwc2_hsotg_process_req_status(struct dwc2_hsotg *hsotg, switch (ctrl->bRequestType & USB_RECIP_MASK) { case USB_RECIP_DEVICE: - reply = cpu_to_le16(0); /* bit 0 => self powered, - * bit 1 => remote wakeup */ + /* + * bit 0 => self powered + * bit 1 => remote wakeup + */ + reply = cpu_to_le16(0); break; case USB_RECIP_INTERFACE: @@ -1099,10 +1612,8 @@ static int dwc2_hsotg_ep_sethalt(struct usb_ep *ep, int value, bool now); */ static struct dwc2_hsotg_req *get_ep_head(struct dwc2_hsotg_ep *hs_ep) { - if (list_empty(&hs_ep->queue)) - return NULL; - - return list_first_entry(&hs_ep->queue, struct dwc2_hsotg_req, queue); + return list_first_entry_or_null(&hs_ep->queue, struct dwc2_hsotg_req, + queue); } /** @@ -1147,7 +1658,7 @@ static void dwc2_gadget_start_next_request(struct dwc2_hsotg_ep *hs_ep) * @ctrl: USB control request */ static int dwc2_hsotg_process_req_feature(struct dwc2_hsotg *hsotg, - struct usb_ctrlrequest *ctrl) + struct usb_ctrlrequest *ctrl) { struct dwc2_hsotg_ep *ep0 = hsotg->eps_out[0]; struct dwc2_hsotg_req *hs_req; @@ -1232,9 +1743,8 @@ static int dwc2_hsotg_process_req_feature(struct dwc2_hsotg *hsotg, } /* If we have pending request, then start it */ - if (!ep->req) { + if (!ep->req) dwc2_gadget_start_next_request(ep); - } } break; @@ -1297,7 +1807,7 @@ static void dwc2_hsotg_stall_ep0(struct dwc2_hsotg *hsotg) * gadget driver). */ static void dwc2_hsotg_process_control(struct dwc2_hsotg *hsotg, - struct usb_ctrlrequest *ctrl) + struct usb_ctrlrequest *ctrl) { struct dwc2_hsotg_ep *ep0 = hsotg->eps_out[0]; int ret = 0; @@ -1373,7 +1883,7 @@ static void dwc2_hsotg_process_control(struct dwc2_hsotg *hsotg, * EP0 setup packets */ static void dwc2_hsotg_complete_setup(struct usb_ep *ep, - struct usb_request *req) + struct usb_request *req) { struct dwc2_hsotg_ep *hs_ep = our_ep(ep); struct dwc2_hsotg *hsotg = hs_ep->parent; @@ -1431,7 +1941,7 @@ static void dwc2_hsotg_enqueue_setup(struct dwc2_hsotg *hsotg) } static void dwc2_hsotg_program_zlp(struct dwc2_hsotg *hsotg, - struct dwc2_hsotg_ep *hs_ep) + struct dwc2_hsotg_ep *hs_ep) { u32 ctrl; u8 index = hs_ep->index; @@ -1440,14 +1950,21 @@ static void dwc2_hsotg_program_zlp(struct dwc2_hsotg *hsotg, if (hs_ep->dir_in) dev_dbg(hsotg->dev, "Sending zero-length packet on ep%d\n", - index); + index); else dev_dbg(hsotg->dev, "Receiving zero-length packet on ep%d\n", - index); + index); + if (using_desc_dma(hsotg)) { + /* Not specific buffer needed for ep0 ZLP */ + dma_addr_t dma = hs_ep->desc_list_dma; - dwc2_writel(DXEPTSIZ_MC(1) | DXEPTSIZ_PKTCNT(1) | - DXEPTSIZ_XFERSIZE(0), hsotg->regs + - epsiz_reg); + dwc2_gadget_set_ep0_desc_chain(hsotg, hs_ep); + dwc2_gadget_config_nonisoc_xfer_ddma(hs_ep, dma, 0); + } else { + dwc2_writel(DXEPTSIZ_MC(1) | DXEPTSIZ_PKTCNT(1) | + DXEPTSIZ_XFERSIZE(0), hsotg->regs + + epsiz_reg); + } ctrl = dwc2_readl(hsotg->regs + epctl_reg); ctrl |= DXEPCTL_CNAK; /* clear NAK set by core */ @@ -1470,11 +1987,10 @@ static void dwc2_hsotg_program_zlp(struct dwc2_hsotg *hsotg, * Note, expects the ep to already be locked as appropriate. */ static void dwc2_hsotg_complete_request(struct dwc2_hsotg *hsotg, - struct dwc2_hsotg_ep *hs_ep, + struct dwc2_hsotg_ep *hs_ep, struct dwc2_hsotg_req *hs_req, int result) { - if (!hs_req) { dev_dbg(hsotg->dev, "%s: nothing to complete?\n", __func__); return; @@ -1510,14 +2026,126 @@ static void dwc2_hsotg_complete_request(struct dwc2_hsotg *hsotg, spin_lock(&hsotg->lock); } + /* In DDMA don't need to proceed to starting of next ISOC request */ + if (using_desc_dma(hsotg) && hs_ep->isochronous) + return; + /* * Look to see if there is anything else to do. Note, the completion * of the previous request may have caused a new request to be started * so be careful when doing this. */ - if (!hs_ep->req && result >= 0) { + if (!hs_ep->req && result >= 0) dwc2_gadget_start_next_request(hs_ep); +} + +/* + * dwc2_gadget_complete_isoc_request_ddma - complete an isoc request in DDMA + * @hs_ep: The endpoint the request was on. + * + * Get first request from the ep queue, determine descriptor on which complete + * happened. SW based on isoc_chain_num discovers which half of the descriptor + * chain is currently in use by HW, adjusts dma_address and calculates index + * of completed descriptor based on the value of DEPDMA register. Update actual + * length of request, giveback to gadget. + */ +static void dwc2_gadget_complete_isoc_request_ddma(struct dwc2_hsotg_ep *hs_ep) +{ + struct dwc2_hsotg *hsotg = hs_ep->parent; + struct dwc2_hsotg_req *hs_req; + struct usb_request *ureq; + int index; + dma_addr_t dma_addr; + u32 dma_reg; + u32 depdma; + u32 desc_sts; + u32 mask; + + hs_req = get_ep_head(hs_ep); + if (!hs_req) { + dev_warn(hsotg->dev, "%s: ISOC EP queue empty\n", __func__); + return; + } + ureq = &hs_req->req; + + dma_addr = hs_ep->desc_list_dma; + + /* + * If lower half of descriptor chain is currently use by SW, + * that means higher half is being processed by HW, so shift + * DMA address to higher half of descriptor chain. + */ + if (!hs_ep->isoc_chain_num) + dma_addr += sizeof(struct dwc2_dma_desc) * + (MAX_DMA_DESC_NUM_GENERIC / 2); + + dma_reg = hs_ep->dir_in ? DIEPDMA(hs_ep->index) : DOEPDMA(hs_ep->index); + depdma = dwc2_readl(hsotg->regs + dma_reg); + + index = (depdma - dma_addr) / sizeof(struct dwc2_dma_desc) - 1; + desc_sts = hs_ep->desc_list[index].status; + + mask = hs_ep->dir_in ? DEV_DMA_ISOC_TX_NBYTES_MASK : + DEV_DMA_ISOC_RX_NBYTES_MASK; + ureq->actual = ureq->length - + ((desc_sts & mask) >> DEV_DMA_ISOC_NBYTES_SHIFT); + + /* Adjust actual length for ISOC Out if length is not align of 4 */ + if (!hs_ep->dir_in && ureq->length & 0x3) + ureq->actual += 4 - (ureq->length & 0x3); + + dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, 0); +} + +/* + * dwc2_gadget_start_next_isoc_ddma - start next isoc request, if any. + * @hs_ep: The isochronous endpoint to be re-enabled. + * + * If ep has been disabled due to last descriptor servicing (IN endpoint) or + * BNA (OUT endpoint) check the status of other half of descriptor chain that + * was under SW control till HW was busy and restart the endpoint if needed. + */ +static void dwc2_gadget_start_next_isoc_ddma(struct dwc2_hsotg_ep *hs_ep) +{ + struct dwc2_hsotg *hsotg = hs_ep->parent; + u32 depctl; + u32 dma_reg; + u32 ctrl; + u32 dma_addr = hs_ep->desc_list_dma; + unsigned char index = hs_ep->index; + + dma_reg = hs_ep->dir_in ? DIEPDMA(index) : DOEPDMA(index); + depctl = hs_ep->dir_in ? DIEPCTL(index) : DOEPCTL(index); + + ctrl = dwc2_readl(hsotg->regs + depctl); + + /* + * EP was disabled if HW has processed last descriptor or BNA was set. + * So restart ep if SW has prepared new descriptor chain in ep_queue + * routine while HW was busy. + */ + if (!(ctrl & DXEPCTL_EPENA)) { + if (!hs_ep->next_desc) { + dev_dbg(hsotg->dev, "%s: No more ISOC requests\n", + __func__); + return; + } + + dma_addr += sizeof(struct dwc2_dma_desc) * + (MAX_DMA_DESC_NUM_GENERIC / 2) * + hs_ep->isoc_chain_num; + dwc2_writel(dma_addr, hsotg->regs + dma_reg); + + ctrl |= DXEPCTL_EPENA | DXEPCTL_CNAK; + dwc2_writel(ctrl, hsotg->regs + depctl); + + /* Switch ISOC descriptor chain number being processed by SW*/ + hs_ep->isoc_chain_num = (hs_ep->isoc_chain_num ^ 1) & 0x1; + hs_ep->next_desc = 0; + + dev_dbg(hsotg->dev, "%s: Restarted isochronous endpoint\n", + __func__); } } @@ -1540,13 +2168,12 @@ static void dwc2_hsotg_rx_data(struct dwc2_hsotg *hsotg, int ep_idx, int size) int max_req; int read_ptr; - if (!hs_req) { u32 epctl = dwc2_readl(hsotg->regs + DOEPCTL(ep_idx)); int ptr; dev_dbg(hsotg->dev, - "%s: FIFO %d bytes on ep%d but no req (DXEPCTl=0x%08x)\n", + "%s: FIFO %d bytes on ep%d but no req (DXEPCTl=0x%08x)\n", __func__, size, ep_idx, epctl); /* dump the data from the FIFO, we've nothing we can do */ @@ -1606,7 +2233,7 @@ static void dwc2_hsotg_ep0_zlp(struct dwc2_hsotg *hsotg, bool dir_in) } static void dwc2_hsotg_change_ep_iso_parity(struct dwc2_hsotg *hsotg, - u32 epctl_reg) + u32 epctl_reg) { u32 ctrl; @@ -1618,6 +2245,36 @@ static void dwc2_hsotg_change_ep_iso_parity(struct dwc2_hsotg *hsotg, dwc2_writel(ctrl, hsotg->regs + epctl_reg); } +/* + * dwc2_gadget_get_xfersize_ddma - get transferred bytes amount from desc + * @hs_ep - The endpoint on which transfer went + * + * Iterate over endpoints descriptor chain and get info on bytes remained + * in DMA descriptors after transfer has completed. Used for non isoc EPs. + */ +static unsigned int dwc2_gadget_get_xfersize_ddma(struct dwc2_hsotg_ep *hs_ep) +{ + struct dwc2_hsotg *hsotg = hs_ep->parent; + unsigned int bytes_rem = 0; + struct dwc2_dma_desc *desc = hs_ep->desc_list; + int i; + u32 status; + + if (!desc) + return -EINVAL; + + for (i = 0; i < hs_ep->desc_count; ++i) { + status = desc->status; + bytes_rem += status & DEV_DMA_NBYTES_MASK; + + if (status & DEV_DMA_STS_MASK) + dev_err(hsotg->dev, "descriptor %d closed with %x\n", + i, status & DEV_DMA_STS_MASK); + } + + return bytes_rem; +} + /** * dwc2_hsotg_handle_outdone - handle receiving OutDone/SetupDone from RXFIFO * @hsotg: The device instance @@ -1633,7 +2290,7 @@ static void dwc2_hsotg_handle_outdone(struct dwc2_hsotg *hsotg, int epnum) struct dwc2_hsotg_ep *hs_ep = hsotg->eps_out[epnum]; struct dwc2_hsotg_req *hs_req = hs_ep->req; struct usb_request *req = &hs_req->req; - unsigned size_left = DXEPTSIZ_XFERSIZE_GET(epsize); + unsigned int size_left = DXEPTSIZ_XFERSIZE_GET(epsize); int result = 0; if (!hs_req) { @@ -1648,8 +2305,11 @@ static void dwc2_hsotg_handle_outdone(struct dwc2_hsotg *hsotg, int epnum) return; } + if (using_desc_dma(hsotg)) + size_left = dwc2_gadget_get_xfersize_ddma(hs_ep); + if (using_dma(hsotg)) { - unsigned size_done; + unsigned int size_done; /* * Calculate the size of the transfer by checking how much @@ -1682,7 +2342,9 @@ static void dwc2_hsotg_handle_outdone(struct dwc2_hsotg *hsotg, int epnum) */ } - if (epnum == 0 && hsotg->ep0_state == DWC2_EP0_DATA_OUT) { + /* DDMA IN status phase will start from StsPhseRcvd interrupt */ + if (!using_desc_dma(hsotg) && epnum == 0 && + hsotg->ep0_state == DWC2_EP0_DATA_OUT) { /* Move to STATUS IN */ dwc2_hsotg_ep0_zlp(hsotg, true); return; @@ -1732,7 +2394,7 @@ static void dwc2_hsotg_handle_rx(struct dwc2_hsotg *hsotg) size >>= GRXSTS_BYTECNT_SHIFT; dev_dbg(hsotg->dev, "%s: GRXSTSP=0x%08x (%d@%d)\n", - __func__, grxstsr, size, epnum); + __func__, grxstsr, size, epnum); switch ((status & GRXSTS_PKTSTS_MASK) >> GRXSTS_PKTSTS_SHIFT) { case GRXSTS_PKTSTS_GLOBALOUTNAK: @@ -1812,17 +2474,17 @@ static u32 dwc2_hsotg_ep0_mps(unsigned int mps) * @hsotg: The driver state. * @ep: The index number of the endpoint * @mps: The maximum packet size in bytes + * @mc: The multicount value * * Configure the maximum packet size for the given endpoint, updating * the hardware control registers to reflect this. */ static void dwc2_hsotg_set_ep_maxpacket(struct dwc2_hsotg *hsotg, - unsigned int ep, unsigned int mps, unsigned int dir_in) + unsigned int ep, unsigned int mps, + unsigned int mc, unsigned int dir_in) { struct dwc2_hsotg_ep *hs_ep; void __iomem *regs = hsotg->regs; - u32 mpsval; - u32 mcval; u32 reg; hs_ep = index_to_ep(hsotg, ep, dir_in); @@ -1830,32 +2492,32 @@ static void dwc2_hsotg_set_ep_maxpacket(struct dwc2_hsotg *hsotg, return; if (ep == 0) { + u32 mps_bytes = mps; + /* EP0 is a special case */ - mpsval = dwc2_hsotg_ep0_mps(mps); - if (mpsval > 3) + mps = dwc2_hsotg_ep0_mps(mps_bytes); + if (mps > 3) goto bad_mps; - hs_ep->ep.maxpacket = mps; + hs_ep->ep.maxpacket = mps_bytes; hs_ep->mc = 1; } else { - mpsval = mps & DXEPCTL_MPS_MASK; - if (mpsval > 1024) + if (mps > 1024) goto bad_mps; - mcval = ((mps >> 11) & 0x3) + 1; - hs_ep->mc = mcval; - if (mcval > 3) + hs_ep->mc = mc; + if (mc > 3) goto bad_mps; - hs_ep->ep.maxpacket = mpsval; + hs_ep->ep.maxpacket = mps; } if (dir_in) { reg = dwc2_readl(regs + DIEPCTL(ep)); reg &= ~DXEPCTL_MPS_MASK; - reg |= mpsval; + reg |= mps; dwc2_writel(reg, regs + DIEPCTL(ep)); } else { reg = dwc2_readl(regs + DOEPCTL(ep)); reg &= ~DXEPCTL_MPS_MASK; - reg |= mpsval; + reg |= mps; dwc2_writel(reg, regs + DOEPCTL(ep)); } @@ -1907,7 +2569,7 @@ static void dwc2_hsotg_txfifo_flush(struct dwc2_hsotg *hsotg, unsigned int idx) * make an attempt to write data into the FIFO. */ static int dwc2_hsotg_trytx(struct dwc2_hsotg *hsotg, - struct dwc2_hsotg_ep *hs_ep) + struct dwc2_hsotg_ep *hs_ep) { struct dwc2_hsotg_req *hs_req = hs_ep->req; @@ -1918,7 +2580,7 @@ static int dwc2_hsotg_trytx(struct dwc2_hsotg *hsotg, */ if (hs_ep->index != 0) dwc2_hsotg_ctrl_epint(hsotg, hs_ep->index, - hs_ep->dir_in, 0); + hs_ep->dir_in, 0); return 0; } @@ -1940,7 +2602,7 @@ static int dwc2_hsotg_trytx(struct dwc2_hsotg *hsotg, * call the relevant completion routines. */ static void dwc2_hsotg_complete_in(struct dwc2_hsotg *hsotg, - struct dwc2_hsotg_ep *hs_ep) + struct dwc2_hsotg_ep *hs_ep) { struct dwc2_hsotg_req *hs_req = hs_ep->req; u32 epsize = dwc2_readl(hsotg->regs + DIEPTSIZ(hs_ep->index)); @@ -1954,6 +2616,13 @@ static void dwc2_hsotg_complete_in(struct dwc2_hsotg *hsotg, /* Finish ZLP handling for IN EP0 transactions */ if (hs_ep->index == 0 && hsotg->ep0_state == DWC2_EP0_STATUS_IN) { dev_dbg(hsotg->dev, "zlp packet sent\n"); + + /* + * While send zlp for DWC2_EP0_STATUS_IN EP direction was + * changed to IN. Change back to complete OUT transfer request + */ + hs_ep->dir_in = 0; + dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, 0); if (hsotg->test_mode) { int ret; @@ -1961,7 +2630,7 @@ static void dwc2_hsotg_complete_in(struct dwc2_hsotg *hsotg, ret = dwc2_hsotg_set_test_mode(hsotg, hsotg->test_mode); if (ret < 0) { dev_dbg(hsotg->dev, "Invalid Test #%d\n", - hsotg->test_mode); + hsotg->test_mode); dwc2_hsotg_stall_ep0(hsotg); return; } @@ -1979,8 +2648,14 @@ static void dwc2_hsotg_complete_in(struct dwc2_hsotg *hsotg, * past the end of the buffer (DMA transfers are always 32bit * aligned). */ - - size_left = DXEPTSIZ_XFERSIZE_GET(epsize); + if (using_desc_dma(hsotg)) { + size_left = dwc2_gadget_get_xfersize_ddma(hs_ep); + if (size_left < 0) + dev_err(hsotg->dev, "error parsing DDMA results %d\n", + size_left); + } else { + size_left = DXEPTSIZ_XFERSIZE_GET(epsize); + } size_done = hs_ep->size_loaded - size_left; size_done += hs_ep->last_load; @@ -2128,12 +2803,28 @@ static void dwc2_gadget_handle_out_token_ep_disabled(struct dwc2_hsotg_ep *ep) struct dwc2_hsotg *hsotg = ep->parent; int dir_in = ep->dir_in; u32 doepmsk; + u32 tmp; if (dir_in || !ep->isochronous) return; + /* + * Store frame in which irq was asserted here, as + * it can change while completing request below. + */ + tmp = dwc2_hsotg_read_frameno(hsotg); + dwc2_hsotg_complete_request(hsotg, ep, get_ep_head(ep), -ENODATA); + if (using_desc_dma(hsotg)) { + if (ep->target_frame == TARGET_FRAME_INITIAL) { + /* Start first ISO Out */ + ep->target_frame = tmp; + dwc2_gadget_start_isoc_ddma(ep); + } + return; + } + if (ep->interval > 1 && ep->target_frame == TARGET_FRAME_INITIAL) { u32 dsts; @@ -2159,19 +2850,19 @@ static void dwc2_gadget_handle_out_token_ep_disabled(struct dwc2_hsotg_ep *ep) } /** -* dwc2_gadget_handle_nak - handle NAK interrupt -* @hs_ep: The endpoint on which interrupt is asserted. -* -* This is starting point for ISOC-IN transfer, synchronization done with -* first IN token received from host while corresponding EP is disabled. -* -* Device does not know when first one token will arrive from host. On first -* token arrival HW generates 2 interrupts: 'in token received while FIFO empty' -* and 'NAK'. NAK interrupt for ISOC-IN means that token has arrived and ZLP was -* sent in response to that as there was no data in FIFO. SW is basing on this -* interrupt to obtain frame in which token has come and then based on the -* interval calculates next frame for transfer. -*/ + * dwc2_gadget_handle_nak - handle NAK interrupt + * @hs_ep: The endpoint on which interrupt is asserted. + * + * This is starting point for ISOC-IN transfer, synchronization done with + * first IN token received from host while corresponding EP is disabled. + * + * Device does not know when first one token will arrive from host. On first + * token arrival HW generates 2 interrupts: 'in token received while FIFO empty' + * and 'NAK'. NAK interrupt for ISOC-IN means that token has arrived and ZLP was + * sent in response to that as there was no data in FIFO. SW is basing on this + * interrupt to obtain frame in which token has come and then based on the + * interval calculates next frame for transfer. + */ static void dwc2_gadget_handle_nak(struct dwc2_hsotg_ep *hs_ep) { struct dwc2_hsotg *hsotg = hs_ep->parent; @@ -2182,6 +2873,12 @@ static void dwc2_gadget_handle_nak(struct dwc2_hsotg_ep *hs_ep) if (hs_ep->target_frame == TARGET_FRAME_INITIAL) { hs_ep->target_frame = dwc2_hsotg_read_frameno(hsotg); + + if (using_desc_dma(hsotg)) { + dwc2_gadget_start_isoc_ddma(hs_ep); + return; + } + if (hs_ep->interval > 1) { u32 ctrl = dwc2_readl(hsotg->regs + DIEPCTL(hs_ep->index)); @@ -2209,7 +2906,7 @@ static void dwc2_gadget_handle_nak(struct dwc2_hsotg_ep *hs_ep) * Process and clear any interrupt pending for an individual endpoint */ static void dwc2_hsotg_epint(struct dwc2_hsotg *hsotg, unsigned int idx, - int dir_in) + int dir_in) { struct dwc2_hsotg_ep *hs_ep = index_to_ep(hsotg, idx, dir_in); u32 epint_reg = dir_in ? DIEPINT(idx) : DOEPINT(idx); @@ -2226,7 +2923,7 @@ static void dwc2_hsotg_epint(struct dwc2_hsotg *hsotg, unsigned int idx, if (!hs_ep) { dev_err(hsotg->dev, "%s:Interrupt for unconfigured ep%d(%s)\n", - __func__, idx, dir_in ? "in" : "out"); + __func__, idx, dir_in ? "in" : "out"); return; } @@ -2237,8 +2934,15 @@ static void dwc2_hsotg_epint(struct dwc2_hsotg *hsotg, unsigned int idx, if (idx == 0 && (ints & (DXEPINT_SETUP | DXEPINT_SETUP_RCVD))) ints &= ~DXEPINT_XFERCOMPL; - if (ints & DXEPINT_STSPHSERCVD) - dev_dbg(hsotg->dev, "%s: StsPhseRcvd asserted\n", __func__); + /* + * Don't process XferCompl interrupt in DDMA if EP0 is still in SETUP + * stage and xfercomplete was generated without SETUP phase done + * interrupt. SW should parse received setup packet only after host's + * exit from setup phase of control transfer. + */ + if (using_desc_dma(hsotg) && idx == 0 && !hs_ep->dir_in && + hsotg->ep0_state == DWC2_EP0_SETUP && !(ints & DXEPINT_SETUP)) + ints &= ~DXEPINT_XFERCOMPL; if (ints & DXEPINT_XFERCOMPL) { dev_dbg(hsotg->dev, @@ -2246,11 +2950,17 @@ static void dwc2_hsotg_epint(struct dwc2_hsotg *hsotg, unsigned int idx, __func__, dwc2_readl(hsotg->regs + epctl_reg), dwc2_readl(hsotg->regs + epsiz_reg)); - /* - * we get OutDone from the FIFO, so we only need to look - * at completing IN requests here - */ - if (dir_in) { + /* In DDMA handle isochronous requests separately */ + if (using_desc_dma(hsotg) && hs_ep->isochronous) { + dwc2_gadget_complete_isoc_request_ddma(hs_ep); + /* Try to start next isoc request */ + dwc2_gadget_start_next_isoc_ddma(hs_ep); + } else if (dir_in) { + /* + * We get OutDone from the FIFO, so we only + * need to look at completing IN requests here + * if operating slave mode + */ if (hs_ep->isochronous && hs_ep->interval > 1) dwc2_gadget_incr_frame_num(hs_ep); @@ -2302,9 +3012,30 @@ static void dwc2_hsotg_epint(struct dwc2_hsotg *hsotg, unsigned int idx, } } + if (ints & DXEPINT_STSPHSERCVD) { + dev_dbg(hsotg->dev, "%s: StsPhseRcvd\n", __func__); + + /* Move to STATUS IN for DDMA */ + if (using_desc_dma(hsotg)) + dwc2_hsotg_ep0_zlp(hsotg, true); + } + if (ints & DXEPINT_BACK2BACKSETUP) dev_dbg(hsotg->dev, "%s: B2BSetup/INEPNakEff\n", __func__); + if (ints & DXEPINT_BNAINTR) { + dev_dbg(hsotg->dev, "%s: BNA interrupt\n", __func__); + + /* + * Try to start next isoc request, if any. + * Sometimes the endpoint remains enabled after BNA interrupt + * assertion, which is not expected, hence we can enter here + * couple of times. + */ + if (hs_ep->isochronous) + dwc2_gadget_start_next_isoc_ddma(hs_ep); + } + if (dir_in && !hs_ep->isochronous) { /* not sure if this is important, but we'll clear it anyway */ if (ints & DXEPINT_INTKNTXFEMP) { @@ -2372,6 +3103,8 @@ static void dwc2_hsotg_irq_enumdone(struct dwc2_hsotg *hsotg) case DSTS_ENUMSPD_LS: hsotg->gadget.speed = USB_SPEED_LOW; + ep0_mps = 8; + ep_mps = 8; /* * note, we don't actually support LS in this driver at the * moment, and the documentation seems to imply that it isn't @@ -2390,13 +3123,15 @@ static void dwc2_hsotg_irq_enumdone(struct dwc2_hsotg *hsotg) if (ep0_mps) { int i; /* Initialize ep0 for both in and out directions */ - dwc2_hsotg_set_ep_maxpacket(hsotg, 0, ep0_mps, 1); - dwc2_hsotg_set_ep_maxpacket(hsotg, 0, ep0_mps, 0); + dwc2_hsotg_set_ep_maxpacket(hsotg, 0, ep0_mps, 0, 1); + dwc2_hsotg_set_ep_maxpacket(hsotg, 0, ep0_mps, 0, 0); for (i = 1; i < hsotg->num_of_eps; i++) { if (hsotg->eps_in[i]) - dwc2_hsotg_set_ep_maxpacket(hsotg, i, ep_mps, 1); + dwc2_hsotg_set_ep_maxpacket(hsotg, i, ep_mps, + 0, 1); if (hsotg->eps_out[i]) - dwc2_hsotg_set_ep_maxpacket(hsotg, i, ep_mps, 0); + dwc2_hsotg_set_ep_maxpacket(hsotg, i, ep_mps, + 0, 0); } } @@ -2423,13 +3158,13 @@ static void kill_all_requests(struct dwc2_hsotg *hsotg, int result) { struct dwc2_hsotg_req *req, *treq; - unsigned size; + unsigned int size; ep->req = NULL; list_for_each_entry_safe(req, treq, &ep->queue, queue) dwc2_hsotg_complete_request(hsotg, ep, req, - result); + result); if (!hsotg->dedicated_fifos) return; @@ -2448,7 +3183,7 @@ static void kill_all_requests(struct dwc2_hsotg *hsotg, */ void dwc2_hsotg_disconnect(struct dwc2_hsotg *hsotg) { - unsigned ep; + unsigned int ep; if (!hsotg->connected) return; @@ -2459,10 +3194,10 @@ void dwc2_hsotg_disconnect(struct dwc2_hsotg *hsotg) for (ep = 0; ep < hsotg->num_of_eps; ep++) { if (hsotg->eps_in[ep]) kill_all_requests(hsotg, hsotg->eps_in[ep], - -ESHUTDOWN); + -ESHUTDOWN); if (hsotg->eps_out[ep]) kill_all_requests(hsotg, hsotg->eps_out[ep], - -ESHUTDOWN); + -ESHUTDOWN); } call_gadget(hsotg, disconnect); @@ -2511,17 +3246,18 @@ static void dwc2_hsotg_irq_fifoempty(struct dwc2_hsotg *hsotg, bool periodic) * Issue a soft reset to the core, and await the core finishing it. */ void dwc2_hsotg_core_init_disconnected(struct dwc2_hsotg *hsotg, - bool is_usb_reset) + bool is_usb_reset) { u32 intmsk; u32 val; u32 usbcfg; + u32 dcfg = 0; /* Kill any ep0 requests as controller will be reinitialized */ kill_all_requests(hsotg, hsotg->eps_out[0], -ECONNRESET); if (!is_usb_reset) - if (dwc2_core_reset(hsotg)) + if (dwc2_core_reset(hsotg, true)) return; /* @@ -2532,12 +3268,19 @@ void dwc2_hsotg_core_init_disconnected(struct dwc2_hsotg *hsotg, /* keep other bits untouched (so e.g. forced modes are not lost) */ usbcfg = dwc2_readl(hsotg->regs + GUSBCFG); usbcfg &= ~(GUSBCFG_TOUTCAL_MASK | GUSBCFG_PHYIF16 | GUSBCFG_SRPCAP | - GUSBCFG_HNPCAP); + GUSBCFG_HNPCAP | GUSBCFG_USBTRDTIM_MASK); - /* set the PLL on, remove the HNP/SRP and set the PHY */ - val = (hsotg->phyif == GUSBCFG_PHYIF8) ? 9 : 5; - usbcfg |= hsotg->phyif | GUSBCFG_TOUTCAL(7) | - (val << GUSBCFG_USBTRDTIM_SHIFT); + if (hsotg->params.phy_type == DWC2_PHY_TYPE_PARAM_FS && + (hsotg->params.speed == DWC2_SPEED_PARAM_FULL || + hsotg->params.speed == DWC2_SPEED_PARAM_LOW)) { + /* FS/LS Dedicated Transceiver Interface */ + usbcfg |= GUSBCFG_PHYSEL; + } else { + /* set the PLL on, remove the HNP/SRP and set the PHY */ + val = (hsotg->phyif == GUSBCFG_PHYIF8) ? 9 : 5; + usbcfg |= hsotg->phyif | GUSBCFG_TOUTCAL(7) | + (val << GUSBCFG_USBTRDTIM_SHIFT); + } dwc2_writel(usbcfg, hsotg->regs + GUSBCFG); dwc2_hsotg_init_fifo(hsotg); @@ -2545,7 +3288,23 @@ void dwc2_hsotg_core_init_disconnected(struct dwc2_hsotg *hsotg, if (!is_usb_reset) __orr32(hsotg->regs + DCTL, DCTL_SFTDISCON); - dwc2_writel(DCFG_EPMISCNT(1) | DCFG_DEVSPD_HS, hsotg->regs + DCFG); + dcfg |= DCFG_EPMISCNT(1); + + switch (hsotg->params.speed) { + case DWC2_SPEED_PARAM_LOW: + dcfg |= DCFG_DEVSPD_LS; + break; + case DWC2_SPEED_PARAM_FULL: + if (hsotg->params.phy_type == DWC2_PHY_TYPE_PARAM_FS) + dcfg |= DCFG_DEVSPD_FS48; + else + dcfg |= DCFG_DEVSPD_FS; + break; + default: + dcfg |= DCFG_DEVSPD_HS; + } + + dwc2_writel(dcfg, hsotg->regs + DCFG); /* Clear any pending OTG interrupts */ dwc2_writel(0xffffffff, hsotg->regs + GOTGINT); @@ -2556,23 +3315,31 @@ void dwc2_hsotg_core_init_disconnected(struct dwc2_hsotg *hsotg, GINTSTS_GOUTNAKEFF | GINTSTS_GINNAKEFF | GINTSTS_USBRST | GINTSTS_RESETDET | GINTSTS_ENUMDONE | GINTSTS_OTGINT | - GINTSTS_USBSUSP | GINTSTS_WKUPINT | - GINTSTS_INCOMPL_SOIN | GINTSTS_INCOMPL_SOOUT; + GINTSTS_USBSUSP | GINTSTS_WKUPINT; + + if (!using_desc_dma(hsotg)) + intmsk |= GINTSTS_INCOMPL_SOIN | GINTSTS_INCOMPL_SOOUT; - if (hsotg->core_params->external_id_pin_ctl <= 0) + if (!hsotg->params.external_id_pin_ctl) intmsk |= GINTSTS_CONIDSTSCHNG; dwc2_writel(intmsk, hsotg->regs + GINTMSK); - if (using_dma(hsotg)) + if (using_dma(hsotg)) { dwc2_writel(GAHBCFG_GLBL_INTR_EN | GAHBCFG_DMA_EN | (GAHBCFG_HBSTLEN_INCR4 << GAHBCFG_HBSTLEN_SHIFT), hsotg->regs + GAHBCFG); - else + + /* Set DDMA mode support in the core if needed */ + if (using_desc_dma(hsotg)) + __orr32(hsotg->regs + DCFG, DCFG_DESCDMA_EN); + + } else { dwc2_writel(((hsotg->dedicated_fifos) ? (GAHBCFG_NP_TXF_EMP_LVL | GAHBCFG_P_TXF_EMP_LVL) : 0) | GAHBCFG_GLBL_INTR_EN, hsotg->regs + GAHBCFG); + } /* * If INTknTXFEmpMsk is enabled, it's important to disable ep interrupts @@ -2588,13 +3355,18 @@ void dwc2_hsotg_core_init_disconnected(struct dwc2_hsotg *hsotg, /* * don't need XferCompl, we get that from RXFIFO in slave mode. In - * DMA mode we may need this. + * DMA mode we may need this and StsPhseRcvd. */ - dwc2_writel((using_dma(hsotg) ? (DIEPMSK_XFERCOMPLMSK) : 0) | + dwc2_writel((using_dma(hsotg) ? (DIEPMSK_XFERCOMPLMSK | + DOEPMSK_STSPHSERCVDMSK) : 0) | DOEPMSK_EPDISBLDMSK | DOEPMSK_AHBERRMSK | - DOEPMSK_SETUPMSK | DOEPMSK_STSPHSERCVDMSK, + DOEPMSK_SETUPMSK, hsotg->regs + DOEPMSK); + /* Enable BNA interrupt for DDMA */ + if (using_desc_dma(hsotg)) + __orr32(hsotg->regs + DOEPMSK, DOEPMSK_BNAMSK); + dwc2_writel(0, hsotg->regs + DAINTMSK); dev_dbg(hsotg->dev, "EP0: DIEPCTL0=0x%08x, DOEPCTL0=0x%08x\n", @@ -2789,7 +3561,6 @@ irq_retry: } if (gintsts & (GINTSTS_USBRST | GINTSTS_RESETDET)) { - u32 usb_status = dwc2_readl(hsotg->regs + GOTGCTL); u32 connected = hsotg->connected; @@ -2928,13 +3699,102 @@ irq_retry: */ if (gintsts & IRQ_RETRY_MASK && --retry_count > 0) - goto irq_retry; + goto irq_retry; spin_unlock(&hsotg->lock); return IRQ_HANDLED; } +static int dwc2_hsotg_wait_bit_set(struct dwc2_hsotg *hs_otg, u32 reg, + u32 bit, u32 timeout) +{ + u32 i; + + for (i = 0; i < timeout; i++) { + if (dwc2_readl(hs_otg->regs + reg) & bit) + return 0; + udelay(1); + } + + return -ETIMEDOUT; +} + +static void dwc2_hsotg_ep_stop_xfr(struct dwc2_hsotg *hsotg, + struct dwc2_hsotg_ep *hs_ep) +{ + u32 epctrl_reg; + u32 epint_reg; + + epctrl_reg = hs_ep->dir_in ? DIEPCTL(hs_ep->index) : + DOEPCTL(hs_ep->index); + epint_reg = hs_ep->dir_in ? DIEPINT(hs_ep->index) : + DOEPINT(hs_ep->index); + + dev_dbg(hsotg->dev, "%s: stopping transfer on %s\n", __func__, + hs_ep->name); + + if (hs_ep->dir_in) { + if (hsotg->dedicated_fifos || hs_ep->periodic) { + __orr32(hsotg->regs + epctrl_reg, DXEPCTL_SNAK); + /* Wait for Nak effect */ + if (dwc2_hsotg_wait_bit_set(hsotg, epint_reg, + DXEPINT_INEPNAKEFF, 100)) + dev_warn(hsotg->dev, + "%s: timeout DIEPINT.NAKEFF\n", + __func__); + } else { + __orr32(hsotg->regs + DCTL, DCTL_SGNPINNAK); + /* Wait for Nak effect */ + if (dwc2_hsotg_wait_bit_set(hsotg, GINTSTS, + GINTSTS_GINNAKEFF, 100)) + dev_warn(hsotg->dev, + "%s: timeout GINTSTS.GINNAKEFF\n", + __func__); + } + } else { + if (!(dwc2_readl(hsotg->regs + GINTSTS) & GINTSTS_GOUTNAKEFF)) + __orr32(hsotg->regs + DCTL, DCTL_SGOUTNAK); + + /* Wait for global nak to take effect */ + if (dwc2_hsotg_wait_bit_set(hsotg, GINTSTS, + GINTSTS_GOUTNAKEFF, 100)) + dev_warn(hsotg->dev, "%s: timeout GINTSTS.GOUTNAKEFF\n", + __func__); + } + + /* Disable ep */ + __orr32(hsotg->regs + epctrl_reg, DXEPCTL_EPDIS | DXEPCTL_SNAK); + + /* Wait for ep to be disabled */ + if (dwc2_hsotg_wait_bit_set(hsotg, epint_reg, DXEPINT_EPDISBLD, 100)) + dev_warn(hsotg->dev, + "%s: timeout DOEPCTL.EPDisable\n", __func__); + + /* Clear EPDISBLD interrupt */ + __orr32(hsotg->regs + epint_reg, DXEPINT_EPDISBLD); + + if (hs_ep->dir_in) { + unsigned short fifo_index; + + if (hsotg->dedicated_fifos || hs_ep->periodic) + fifo_index = hs_ep->fifo_index; + else + fifo_index = 0; + + /* Flush TX FIFO */ + dwc2_flush_tx_fifo(hsotg, fifo_index); + + /* Clear Global In NP NAK in Shared FIFO for non periodic ep */ + if (!hsotg->dedicated_fifos && !hs_ep->periodic) + __orr32(hsotg->regs + DCTL, DCTL_CGNPINNAK); + + } else { + /* Remove global NAKs */ + __orr32(hsotg->regs + DCTL, DCTL_CGOUTNAK); + } +} + /** * dwc2_hsotg_ep_enable - enable the given endpoint * @ep: The USB endpint to configure @@ -2943,7 +3803,7 @@ irq_retry: * This is called from the USB gadget code's usb_ep_enable(). */ static int dwc2_hsotg_ep_enable(struct usb_ep *ep, - const struct usb_endpoint_descriptor *desc) + const struct usb_endpoint_descriptor *desc) { struct dwc2_hsotg_ep *hs_ep = our_ep(ep); struct dwc2_hsotg *hsotg = hs_ep->parent; @@ -2952,6 +3812,7 @@ static int dwc2_hsotg_ep_enable(struct usb_ep *ep, u32 epctrl_reg; u32 epctrl; u32 mps; + u32 mc; u32 mask; unsigned int dir_in; unsigned int i, val, size; @@ -2975,6 +3836,7 @@ static int dwc2_hsotg_ep_enable(struct usb_ep *ep, } mps = usb_endpoint_maxp(desc); + mc = usb_endpoint_maxp_mult(desc); /* note, we handle this here instead of dwc2_hsotg_set_ep_maxpacket */ @@ -2984,6 +3846,18 @@ static int dwc2_hsotg_ep_enable(struct usb_ep *ep, dev_dbg(hsotg->dev, "%s: read DxEPCTL=0x%08x from 0x%08x\n", __func__, epctrl, epctrl_reg); + /* Allocate DMA descriptor chain for non-ctrl endpoints */ + if (using_desc_dma(hsotg) && !hs_ep->desc_list) { + hs_ep->desc_list = dmam_alloc_coherent(hsotg->dev, + MAX_DMA_DESC_NUM_GENERIC * + sizeof(struct dwc2_dma_desc), + &hs_ep->desc_list_dma, GFP_ATOMIC); + if (!hs_ep->desc_list) { + ret = -ENOMEM; + goto error2; + } + } + spin_lock_irqsave(&hsotg->lock, flags); epctrl &= ~(DXEPCTL_EPTYPE_MASK | DXEPCTL_MPS_MASK); @@ -2996,7 +3870,7 @@ static int dwc2_hsotg_ep_enable(struct usb_ep *ep, epctrl |= DXEPCTL_USBACTEP; /* update the endpoint state */ - dwc2_hsotg_set_ep_maxpacket(hsotg, hs_ep->index, mps, dir_in); + dwc2_hsotg_set_ep_maxpacket(hsotg, hs_ep->index, mps, mc, dir_in); /* default, set to non-periodic */ hs_ep->isochronous = 0; @@ -3011,6 +3885,8 @@ static int dwc2_hsotg_ep_enable(struct usb_ep *ep, hs_ep->isochronous = 1; hs_ep->interval = 1 << (desc->bInterval - 1); hs_ep->target_frame = TARGET_FRAME_INITIAL; + hs_ep->isoc_chain_num = 0; + hs_ep->next_desc = 0; if (dir_in) { hs_ep->periodic = 1; mask = dwc2_readl(hsotg->regs + DIEPMSK); @@ -3049,12 +3925,13 @@ static int dwc2_hsotg_ep_enable(struct usb_ep *ep, if (dir_in && hsotg->dedicated_fifos) { u32 fifo_index = 0; u32 fifo_size = UINT_MAX; - size = hs_ep->ep.maxpacket*hs_ep->mc; + + size = hs_ep->ep.maxpacket * hs_ep->mc; for (i = 1; i < hsotg->num_of_eps; ++i) { - if (hsotg->fifo_map & (1<<i)) + if (hsotg->fifo_map & (1 << i)) continue; val = dwc2_readl(hsotg->regs + DPTXFSIZN(i)); - val = (val >> FIFOSIZE_DEPTH_SHIFT)*4; + val = (val >> FIFOSIZE_DEPTH_SHIFT) * 4; if (val < size) continue; /* Search for smallest acceptable fifo */ @@ -3067,7 +3944,7 @@ static int dwc2_hsotg_ep_enable(struct usb_ep *ep, dev_err(hsotg->dev, "%s: No suitable fifo found\n", __func__); ret = -ENOMEM; - goto error; + goto error1; } hsotg->fifo_map |= 1 << fifo_index; epctrl |= DXEPCTL_TXFNUM(fifo_index); @@ -3089,8 +3966,17 @@ static int dwc2_hsotg_ep_enable(struct usb_ep *ep, /* enable the endpoint interrupt */ dwc2_hsotg_ctrl_epint(hsotg, index, dir_in, 1); -error: +error1: spin_unlock_irqrestore(&hsotg->lock, flags); + +error2: + if (ret && using_desc_dma(hsotg) && hs_ep->desc_list) { + dmam_free_coherent(hsotg->dev, MAX_DMA_DESC_NUM_GENERIC * + sizeof(struct dwc2_dma_desc), + hs_ep->desc_list, hs_ep->desc_list_dma); + hs_ep->desc_list = NULL; + } + return ret; } @@ -3120,6 +4006,10 @@ static int dwc2_hsotg_ep_disable(struct usb_ep *ep) spin_lock_irqsave(&hsotg->lock, flags); ctrl = dwc2_readl(hsotg->regs + epctrl_reg); + + if (ctrl & DXEPCTL_EPENA) + dwc2_hsotg_ep_stop_xfr(hsotg, hs_ep); + ctrl &= ~DXEPCTL_EPENA; ctrl &= ~DXEPCTL_USBACTEP; ctrl |= DXEPCTL_SNAK; @@ -3158,77 +4048,6 @@ static bool on_list(struct dwc2_hsotg_ep *ep, struct dwc2_hsotg_req *test) return false; } -static int dwc2_hsotg_wait_bit_set(struct dwc2_hsotg *hs_otg, u32 reg, - u32 bit, u32 timeout) -{ - u32 i; - - for (i = 0; i < timeout; i++) { - if (dwc2_readl(hs_otg->regs + reg) & bit) - return 0; - udelay(1); - } - - return -ETIMEDOUT; -} - -static void dwc2_hsotg_ep_stop_xfr(struct dwc2_hsotg *hsotg, - struct dwc2_hsotg_ep *hs_ep) -{ - u32 epctrl_reg; - u32 epint_reg; - - epctrl_reg = hs_ep->dir_in ? DIEPCTL(hs_ep->index) : - DOEPCTL(hs_ep->index); - epint_reg = hs_ep->dir_in ? DIEPINT(hs_ep->index) : - DOEPINT(hs_ep->index); - - dev_dbg(hsotg->dev, "%s: stopping transfer on %s\n", __func__, - hs_ep->name); - if (hs_ep->dir_in) { - __orr32(hsotg->regs + epctrl_reg, DXEPCTL_SNAK); - /* Wait for Nak effect */ - if (dwc2_hsotg_wait_bit_set(hsotg, epint_reg, - DXEPINT_INEPNAKEFF, 100)) - dev_warn(hsotg->dev, - "%s: timeout DIEPINT.NAKEFF\n", __func__); - } else { - if (!(dwc2_readl(hsotg->regs + GINTSTS) & GINTSTS_GOUTNAKEFF)) - __orr32(hsotg->regs + DCTL, DCTL_SGOUTNAK); - - /* Wait for global nak to take effect */ - if (dwc2_hsotg_wait_bit_set(hsotg, GINTSTS, - GINTSTS_GOUTNAKEFF, 100)) - dev_warn(hsotg->dev, - "%s: timeout GINTSTS.GOUTNAKEFF\n", __func__); - } - - /* Disable ep */ - __orr32(hsotg->regs + epctrl_reg, DXEPCTL_EPDIS | DXEPCTL_SNAK); - - /* Wait for ep to be disabled */ - if (dwc2_hsotg_wait_bit_set(hsotg, epint_reg, DXEPINT_EPDISBLD, 100)) - dev_warn(hsotg->dev, - "%s: timeout DOEPCTL.EPDisable\n", __func__); - - if (hs_ep->dir_in) { - if (hsotg->dedicated_fifos) { - dwc2_writel(GRSTCTL_TXFNUM(hs_ep->fifo_index) | - GRSTCTL_TXFFLSH, hsotg->regs + GRSTCTL); - /* Wait for fifo flush */ - if (dwc2_hsotg_wait_bit_set(hsotg, GRSTCTL, - GRSTCTL_TXFFLSH, 100)) - dev_warn(hsotg->dev, - "%s: timeout flushing fifos\n", - __func__); - } - /* TODO: Flush shared tx fifo */ - } else { - /* Remove global NAKs */ - __bic32(hsotg->regs + DCTL, DCTL_SGOUTNAK); - } -} - /** * dwc2_hsotg_ep_dequeue - dequeue given endpoint * @ep: The endpoint to dequeue. @@ -3313,23 +4132,22 @@ static int dwc2_hsotg_ep_sethalt(struct usb_ep *ep, int value, bool now) epctl &= ~DXEPCTL_STALL; xfertype = epctl & DXEPCTL_EPTYPE_MASK; if (xfertype == DXEPCTL_EPTYPE_BULK || - xfertype == DXEPCTL_EPTYPE_INTERRUPT) - epctl |= DXEPCTL_SETD0PID; + xfertype == DXEPCTL_EPTYPE_INTERRUPT) + epctl |= DXEPCTL_SETD0PID; } dwc2_writel(epctl, hs->regs + epreg); } else { - epreg = DOEPCTL(index); epctl = dwc2_readl(hs->regs + epreg); - if (value) + if (value) { epctl |= DXEPCTL_STALL; - else { + } else { epctl &= ~DXEPCTL_STALL; xfertype = epctl & DXEPCTL_EPTYPE_MASK; if (xfertype == DXEPCTL_EPTYPE_BULK || - xfertype == DXEPCTL_EPTYPE_INTERRUPT) - epctl |= DXEPCTL_SETD0PID; + xfertype == DXEPCTL_EPTYPE_INTERRUPT) + epctl |= DXEPCTL_SETD0PID; } dwc2_writel(epctl, hs->regs + epreg); } @@ -3370,7 +4188,7 @@ static struct usb_ep_ops dwc2_hsotg_ep_ops = { }; /** - * dwc2_hsotg_init - initalize the usb core + * dwc2_hsotg_init - initialize the usb core * @hsotg: The driver state */ static void dwc2_hsotg_init(struct dwc2_hsotg *hsotg) @@ -3403,7 +4221,7 @@ static void dwc2_hsotg_init(struct dwc2_hsotg *hsotg) /* keep other bits untouched (so e.g. forced modes are not lost) */ usbcfg = dwc2_readl(hsotg->regs + GUSBCFG); usbcfg &= ~(GUSBCFG_TOUTCAL_MASK | GUSBCFG_PHYIF16 | GUSBCFG_SRPCAP | - GUSBCFG_HNPCAP); + GUSBCFG_HNPCAP | GUSBCFG_USBTRDTIM_MASK); /* set the PLL on, remove the HNP/SRP and set the PHY */ trdtim = (hsotg->phyif == GUSBCFG_PHYIF8) ? 9 : 5; @@ -3424,7 +4242,7 @@ static void dwc2_hsotg_init(struct dwc2_hsotg *hsotg) * to work. */ static int dwc2_hsotg_udc_start(struct usb_gadget *gadget, - struct usb_gadget_driver *driver) + struct usb_gadget_driver *driver) { struct dwc2_hsotg *hsotg = to_hsotg(gadget); unsigned long flags; @@ -3547,7 +4365,7 @@ static int dwc2_hsotg_pullup(struct usb_gadget *gadget, int is_on) unsigned long flags = 0; dev_dbg(hsotg->dev, "%s: is_on: %d op_state: %d\n", __func__, is_on, - hsotg->op_state); + hsotg->op_state); /* Don't modify pullup state while in host mode */ if (hsotg->op_state != OTG_STATE_B_PERIPHERAL) { @@ -3609,7 +4427,7 @@ static int dwc2_hsotg_vbus_session(struct usb_gadget *gadget, int is_active) * * Report how much power the device may consume to the phy. */ -static int dwc2_hsotg_vbus_draw(struct usb_gadget *gadget, unsigned mA) +static int dwc2_hsotg_vbus_draw(struct usb_gadget *gadget, unsigned int mA) { struct dwc2_hsotg *hsotg = to_hsotg(gadget); @@ -3638,7 +4456,7 @@ static const struct usb_gadget_ops dwc2_hsotg_gadget_ops = { * direction information and other state that may be required. */ static void dwc2_hsotg_initep(struct dwc2_hsotg *hsotg, - struct dwc2_hsotg_ep *hs_ep, + struct dwc2_hsotg_ep *hs_ep, int epnum, bool dir_in) { @@ -3665,14 +4483,21 @@ static void dwc2_hsotg_initep(struct dwc2_hsotg *hsotg, hs_ep->parent = hsotg; hs_ep->ep.name = hs_ep->name; - usb_ep_set_maxpacket_limit(&hs_ep->ep, epnum ? 1024 : EP0_MPS_LIMIT); + + if (hsotg->params.speed == DWC2_SPEED_PARAM_LOW) + usb_ep_set_maxpacket_limit(&hs_ep->ep, 8); + else + usb_ep_set_maxpacket_limit(&hs_ep->ep, + epnum ? 1024 : EP0_MPS_LIMIT); hs_ep->ep.ops = &dwc2_hsotg_ep_ops; if (epnum == 0) { hs_ep->ep.caps.type_control = true; } else { - hs_ep->ep.caps.type_iso = true; - hs_ep->ep.caps.type_bulk = true; + if (hsotg->params.speed != DWC2_SPEED_PARAM_LOW) { + hs_ep->ep.caps.type_iso = true; + hs_ep->ep.caps.type_bulk = true; + } hs_ep->ep.caps.type_int = true; } @@ -3688,6 +4513,7 @@ static void dwc2_hsotg_initep(struct dwc2_hsotg *hsotg, if (using_dma(hsotg)) { u32 next = DXEPCTL_NEXTEP((epnum + 1) % 15); + if (dir_in) dwc2_writel(next, hsotg->regs + DIEPCTL(epnum)); else @@ -3714,8 +4540,9 @@ static int dwc2_hsotg_hw_cfg(struct dwc2_hsotg *hsotg) /* Add ep0 */ hsotg->num_of_eps++; - hsotg->eps_in[0] = devm_kzalloc(hsotg->dev, sizeof(struct dwc2_hsotg_ep), - GFP_KERNEL); + hsotg->eps_in[0] = devm_kzalloc(hsotg->dev, + sizeof(struct dwc2_hsotg_ep), + GFP_KERNEL); if (!hsotg->eps_in[0]) return -ENOMEM; /* Same dwc2_hsotg_ep is used in both directions for ep0 */ @@ -3794,7 +4621,6 @@ static void dwc2_hsotg_dump(struct dwc2_hsotg *hsotg) idx, dwc2_readl(regs + DOEPCTL(idx)), dwc2_readl(regs + DOEPTSIZ(idx)), dwc2_readl(regs + DOEPDMA(idx))); - } dev_info(dev, "DVBUSDIS=0x%08x, DVBUSPULSE=%08x\n", @@ -3802,51 +4628,6 @@ static void dwc2_hsotg_dump(struct dwc2_hsotg *hsotg) #endif } -#ifdef CONFIG_OF -static void dwc2_hsotg_of_probe(struct dwc2_hsotg *hsotg) -{ - struct device_node *np = hsotg->dev->of_node; - u32 len = 0; - u32 i = 0; - - /* Enable dma if requested in device tree */ - hsotg->g_using_dma = of_property_read_bool(np, "g-use-dma"); - - /* - * Register TX periodic fifo size per endpoint. - * EP0 is excluded since it has no fifo configuration. - */ - if (!of_find_property(np, "g-tx-fifo-size", &len)) - goto rx_fifo; - - len /= sizeof(u32); - - /* Read tx fifo sizes other than ep0 */ - if (of_property_read_u32_array(np, "g-tx-fifo-size", - &hsotg->g_tx_fifo_sz[1], len)) - goto rx_fifo; - - /* Add ep0 */ - len++; - - /* Make remaining TX fifos unavailable */ - if (len < MAX_EPS_CHANNELS) { - for (i = len; i < MAX_EPS_CHANNELS; i++) - hsotg->g_tx_fifo_sz[i] = 0; - } - -rx_fifo: - /* Register RX fifo size */ - of_property_read_u32(np, "g-rx-fifo-size", &hsotg->g_rx_fifo_sz); - - /* Register NPTX fifo size */ - of_property_read_u32(np, "g-np-tx-fifo-size", - &hsotg->g_np_g_tx_fifo_sz); -} -#else -static inline void dwc2_hsotg_of_probe(struct dwc2_hsotg *hsotg) { } -#endif - /** * dwc2_gadget_init - init function for gadget * @dwc2: The data structure for the DWC2 driver. @@ -3857,33 +4638,11 @@ int dwc2_gadget_init(struct dwc2_hsotg *hsotg, int irq) struct device *dev = hsotg->dev; int epnum; int ret; - int i; - u32 p_tx_fifo[] = DWC2_G_P_LEGACY_TX_FIFO_SIZE; - - /* Initialize to legacy fifo configuration values */ - hsotg->g_rx_fifo_sz = 2048; - hsotg->g_np_g_tx_fifo_sz = 1024; - memcpy(&hsotg->g_tx_fifo_sz[1], p_tx_fifo, sizeof(p_tx_fifo)); - /* Device tree specific probe */ - dwc2_hsotg_of_probe(hsotg); - - /* Check against largest possible value. */ - if (hsotg->g_np_g_tx_fifo_sz > - hsotg->hw_params.dev_nperio_tx_fifo_size) { - dev_warn(dev, "Specified GNPTXFDEP=%d > %d\n", - hsotg->g_np_g_tx_fifo_sz, - hsotg->hw_params.dev_nperio_tx_fifo_size); - hsotg->g_np_g_tx_fifo_sz = - hsotg->hw_params.dev_nperio_tx_fifo_size; - } /* Dump fifo information */ dev_dbg(dev, "NonPeriodic TXFIFO size: %d\n", - hsotg->g_np_g_tx_fifo_sz); - dev_dbg(dev, "RXFIFO size: %d\n", hsotg->g_rx_fifo_sz); - for (i = 0; i < MAX_EPS_CHANNELS; i++) - dev_dbg(dev, "Periodic TXFIFO%2d size: %d\n", i, - hsotg->g_tx_fifo_sz[i]); + hsotg->params.g_np_tx_fifo_size); + dev_dbg(dev, "RXFIFO size: %d\n", hsotg->params.g_rx_fifo_size); hsotg->gadget.max_speed = USB_SPEED_HIGH; hsotg->gadget.ops = &dwc2_hsotg_gadget_ops; @@ -3909,8 +4668,14 @@ int dwc2_gadget_init(struct dwc2_hsotg *hsotg, int irq) if (!hsotg->ep0_buff) return -ENOMEM; + if (using_desc_dma(hsotg)) { + ret = dwc2_gadget_alloc_ctrl_desc_chains(hsotg); + if (ret < 0) + return ret; + } + ret = devm_request_irq(hsotg->dev, irq, dwc2_hsotg_irq, IRQF_SHARED, - dev_name(hsotg->dev), hsotg); + dev_name(hsotg->dev), hsotg); if (ret < 0) { dev_err(dev, "cannot claim IRQ for gadget\n"); return ret; @@ -3941,10 +4706,10 @@ int dwc2_gadget_init(struct dwc2_hsotg *hsotg, int irq) for (epnum = 0; epnum < hsotg->num_of_eps; epnum++) { if (hsotg->eps_in[epnum]) dwc2_hsotg_initep(hsotg, hsotg->eps_in[epnum], - epnum, 1); + epnum, 1); if (hsotg->eps_out[epnum]) dwc2_hsotg_initep(hsotg, hsotg->eps_out[epnum], - epnum, 0); + epnum, 0); } ret = usb_add_gadget_udc(dev, &hsotg->gadget); |