Loading...
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 | /* SPDX-License-Identifier: GPL-2.0 */ /* * Driver for the Synopsys DesignWare AHB DMA Controller * * Copyright (C) 2005-2007 Atmel Corporation * Copyright (C) 2010-2011 ST Microelectronics * Copyright (C) 2016 Intel Corporation */ #include <linux/bitops.h> #include <linux/interrupt.h> #include <linux/dmaengine.h> #include <linux/io-64-nonatomic-hi-lo.h> #include "internal.h" #define DW_DMA_MAX_NR_REQUESTS 16 /* flow controller */ enum dw_dma_fc { DW_DMA_FC_D_M2M, DW_DMA_FC_D_M2P, DW_DMA_FC_D_P2M, DW_DMA_FC_D_P2P, DW_DMA_FC_P_P2M, DW_DMA_FC_SP_P2P, DW_DMA_FC_P_M2P, DW_DMA_FC_DP_P2P, }; /* * Redefine this macro to handle differences between 32- and 64-bit * addressing, big vs. little endian, etc. */ #define DW_REG(name) u32 name; u32 __pad_##name /* Hardware register definitions. */ struct dw_dma_chan_regs { DW_REG(SAR); /* Source Address Register */ DW_REG(DAR); /* Destination Address Register */ DW_REG(LLP); /* Linked List Pointer */ u32 CTL_LO; /* Control Register Low */ u32 CTL_HI; /* Control Register High */ DW_REG(SSTAT); DW_REG(DSTAT); DW_REG(SSTATAR); DW_REG(DSTATAR); u32 CFG_LO; /* Configuration Register Low */ u32 CFG_HI; /* Configuration Register High */ DW_REG(SGR); DW_REG(DSR); }; struct dw_dma_irq_regs { DW_REG(XFER); DW_REG(BLOCK); DW_REG(SRC_TRAN); DW_REG(DST_TRAN); DW_REG(ERROR); }; struct dw_dma_regs { /* per-channel registers */ struct dw_dma_chan_regs CHAN[DW_DMA_MAX_NR_CHANNELS]; /* irq handling */ struct dw_dma_irq_regs RAW; /* r */ struct dw_dma_irq_regs STATUS; /* r (raw & mask) */ struct dw_dma_irq_regs MASK; /* rw (set = irq enabled) */ struct dw_dma_irq_regs CLEAR; /* w (ack, affects "raw") */ DW_REG(STATUS_INT); /* r */ /* software handshaking */ DW_REG(REQ_SRC); DW_REG(REQ_DST); DW_REG(SGL_REQ_SRC); DW_REG(SGL_REQ_DST); DW_REG(LAST_SRC); DW_REG(LAST_DST); /* miscellaneous */ DW_REG(CFG); DW_REG(CH_EN); DW_REG(ID); DW_REG(TEST); /* iDMA 32-bit support */ DW_REG(CLASS_PRIORITY0); DW_REG(CLASS_PRIORITY1); /* optional encoded params, 0x3c8..0x3f7 */ u32 __reserved; /* per-channel configuration registers */ u32 DWC_PARAMS[DW_DMA_MAX_NR_CHANNELS]; u32 MULTI_BLK_TYPE; u32 MAX_BLK_SIZE; /* top-level parameters */ u32 DW_PARAMS; /* component ID */ u32 COMP_TYPE; u32 COMP_VERSION; /* iDMA 32-bit support */ DW_REG(FIFO_PARTITION0); DW_REG(FIFO_PARTITION1); DW_REG(SAI_ERR); DW_REG(GLOBAL_CFG); }; /* Bitfields in DW_PARAMS */ #define DW_PARAMS_NR_CHAN 8 /* number of channels */ #define DW_PARAMS_NR_MASTER 11 /* number of AHB masters */ #define DW_PARAMS_DATA_WIDTH(n) (15 + 2 * (n)) #define DW_PARAMS_DATA_WIDTH1 15 /* master 1 data width */ #define DW_PARAMS_DATA_WIDTH2 17 /* master 2 data width */ #define DW_PARAMS_DATA_WIDTH3 19 /* master 3 data width */ #define DW_PARAMS_DATA_WIDTH4 21 /* master 4 data width */ #define DW_PARAMS_EN 28 /* encoded parameters */ /* Bitfields in DWC_PARAMS */ #define DWC_PARAMS_MBLK_EN 11 /* multi block transfer */ #define DWC_PARAMS_HC_LLP 13 /* set LLP register to zero */ #define DWC_PARAMS_MSIZE 16 /* max group transaction size */ /* bursts size */ enum dw_dma_msize { DW_DMA_MSIZE_1, DW_DMA_MSIZE_4, DW_DMA_MSIZE_8, DW_DMA_MSIZE_16, DW_DMA_MSIZE_32, DW_DMA_MSIZE_64, DW_DMA_MSIZE_128, DW_DMA_MSIZE_256, }; /* Bitfields in LLP */ #define DWC_LLP_LMS(x) ((x) & 3) /* list master select */ #define DWC_LLP_LOC(x) ((x) & ~3) /* next lli */ /* Bitfields in CTL_LO */ #define DWC_CTLL_INT_EN (1 << 0) /* irqs enabled? */ #define DWC_CTLL_DST_WIDTH(n) ((n)<<1) /* bytes per element */ #define DWC_CTLL_SRC_WIDTH(n) ((n)<<4) #define DWC_CTLL_DST_INC (0<<7) /* DAR update/not */ #define DWC_CTLL_DST_DEC (1<<7) #define DWC_CTLL_DST_FIX (2<<7) #define DWC_CTLL_SRC_INC (0<<9) /* SAR update/not */ #define DWC_CTLL_SRC_DEC (1<<9) #define DWC_CTLL_SRC_FIX (2<<9) #define DWC_CTLL_DST_MSIZE(n) ((n)<<11) /* burst, #elements */ #define DWC_CTLL_SRC_MSIZE(n) ((n)<<14) #define DWC_CTLL_S_GATH_EN (1 << 17) /* src gather, !FIX */ #define DWC_CTLL_D_SCAT_EN (1 << 18) /* dst scatter, !FIX */ #define DWC_CTLL_FC(n) ((n) << 20) #define DWC_CTLL_FC_M2M (0 << 20) /* mem-to-mem */ #define DWC_CTLL_FC_M2P (1 << 20) /* mem-to-periph */ #define DWC_CTLL_FC_P2M (2 << 20) /* periph-to-mem */ #define DWC_CTLL_FC_P2P (3 << 20) /* periph-to-periph */ /* plus 4 transfer types for peripheral-as-flow-controller */ #define DWC_CTLL_DMS(n) ((n)<<23) /* dst master select */ #define DWC_CTLL_SMS(n) ((n)<<25) /* src master select */ #define DWC_CTLL_LLP_D_EN (1 << 27) /* dest block chain */ #define DWC_CTLL_LLP_S_EN (1 << 28) /* src block chain */ /* Bitfields in CTL_HI */ #define DWC_CTLH_BLOCK_TS_MASK GENMASK(11, 0) #define DWC_CTLH_BLOCK_TS(x) ((x) & DWC_CTLH_BLOCK_TS_MASK) #define DWC_CTLH_DONE (1 << 12) /* Bitfields in CFG_LO */ #define DWC_CFGL_CH_PRIOR_MASK (0x7 << 5) /* priority mask */ #define DWC_CFGL_CH_PRIOR(x) ((x) << 5) /* priority */ #define DWC_CFGL_CH_SUSP (1 << 8) /* pause xfer */ #define DWC_CFGL_FIFO_EMPTY (1 << 9) /* pause xfer */ #define DWC_CFGL_HS_DST (1 << 10) /* handshake w/dst */ #define DWC_CFGL_HS_SRC (1 << 11) /* handshake w/src */ #define DWC_CFGL_LOCK_CH_XFER (0 << 12) /* scope of LOCK_CH */ #define DWC_CFGL_LOCK_CH_BLOCK (1 << 12) #define DWC_CFGL_LOCK_CH_XACT (2 << 12) #define DWC_CFGL_LOCK_BUS_XFER (0 << 14) /* scope of LOCK_BUS */ #define DWC_CFGL_LOCK_BUS_BLOCK (1 << 14) #define DWC_CFGL_LOCK_BUS_XACT (2 << 14) #define DWC_CFGL_LOCK_CH (1 << 15) /* channel lockout */ #define DWC_CFGL_LOCK_BUS (1 << 16) /* busmaster lockout */ #define DWC_CFGL_HS_DST_POL (1 << 18) /* dst handshake active low */ #define DWC_CFGL_HS_SRC_POL (1 << 19) /* src handshake active low */ #define DWC_CFGL_MAX_BURST(x) ((x) << 20) #define DWC_CFGL_RELOAD_SAR (1 << 30) #define DWC_CFGL_RELOAD_DAR (1 << 31) /* Bitfields in CFG_HI */ #define DWC_CFGH_FCMODE (1 << 0) #define DWC_CFGH_FIFO_MODE (1 << 1) #define DWC_CFGH_PROTCTL(x) ((x) << 2) #define DWC_CFGH_PROTCTL_DATA (0 << 2) /* data access - always set */ #define DWC_CFGH_PROTCTL_PRIV (1 << 2) /* privileged -> AHB HPROT[1] */ #define DWC_CFGH_PROTCTL_BUFFER (2 << 2) /* bufferable -> AHB HPROT[2] */ #define DWC_CFGH_PROTCTL_CACHE (4 << 2) /* cacheable -> AHB HPROT[3] */ #define DWC_CFGH_DS_UPD_EN (1 << 5) #define DWC_CFGH_SS_UPD_EN (1 << 6) #define DWC_CFGH_SRC_PER(x) ((x) << 7) #define DWC_CFGH_DST_PER(x) ((x) << 11) /* Bitfields in SGR */ #define DWC_SGR_SGI(x) ((x) << 0) #define DWC_SGR_SGC(x) ((x) << 20) /* Bitfields in DSR */ #define DWC_DSR_DSI(x) ((x) << 0) #define DWC_DSR_DSC(x) ((x) << 20) /* Bitfields in CFG */ #define DW_CFG_DMA_EN (1 << 0) /* iDMA 32-bit support */ /* bursts size */ enum idma32_msize { IDMA32_MSIZE_1, IDMA32_MSIZE_2, IDMA32_MSIZE_4, IDMA32_MSIZE_8, IDMA32_MSIZE_16, IDMA32_MSIZE_32, }; /* Bitfields in CTL_HI */ #define IDMA32C_CTLH_BLOCK_TS_MASK GENMASK(16, 0) #define IDMA32C_CTLH_BLOCK_TS(x) ((x) & IDMA32C_CTLH_BLOCK_TS_MASK) #define IDMA32C_CTLH_DONE (1 << 17) /* Bitfields in CFG_LO */ #define IDMA32C_CFGL_DST_BURST_ALIGN (1 << 0) /* dst burst align */ #define IDMA32C_CFGL_SRC_BURST_ALIGN (1 << 1) /* src burst align */ #define IDMA32C_CFGL_CH_DRAIN (1 << 10) /* drain FIFO */ #define IDMA32C_CFGL_DST_OPT_BL (1 << 20) /* optimize dst burst length */ #define IDMA32C_CFGL_SRC_OPT_BL (1 << 21) /* optimize src burst length */ /* Bitfields in CFG_HI */ #define IDMA32C_CFGH_SRC_PER(x) ((x) << 0) #define IDMA32C_CFGH_DST_PER(x) ((x) << 4) #define IDMA32C_CFGH_RD_ISSUE_THD(x) ((x) << 8) #define IDMA32C_CFGH_RW_ISSUE_THD(x) ((x) << 18) #define IDMA32C_CFGH_SRC_PER_EXT(x) ((x) << 28) /* src peripheral extension */ #define IDMA32C_CFGH_DST_PER_EXT(x) ((x) << 30) /* dst peripheral extension */ /* Bitfields in FIFO_PARTITION */ #define IDMA32C_FP_PSIZE_CH0(x) ((x) << 0) #define IDMA32C_FP_PSIZE_CH1(x) ((x) << 13) #define IDMA32C_FP_UPDATE (1 << 26) enum dw_dmac_flags { DW_DMA_IS_CYCLIC = 0, DW_DMA_IS_SOFT_LLP = 1, DW_DMA_IS_PAUSED = 2, DW_DMA_IS_INITIALIZED = 3, }; struct dw_dma_chan { struct dma_chan chan; void __iomem *ch_regs; u8 mask; u8 priority; enum dma_transfer_direction direction; /* software emulation of the LLP transfers */ struct list_head *tx_node_active; spinlock_t lock; /* these other elements are all protected by lock */ unsigned long flags; struct list_head active_list; struct list_head queue; unsigned int descs_allocated; /* hardware configuration */ unsigned int block_size; bool nollp; u32 max_burst; /* custom slave configuration */ struct dw_dma_slave dws; /* configuration passed via .device_config */ struct dma_slave_config dma_sconfig; }; static inline struct dw_dma_chan_regs __iomem * __dwc_regs(struct dw_dma_chan *dwc) { return dwc->ch_regs; } #define channel_readl(dwc, name) \ readl(&(__dwc_regs(dwc)->name)) #define channel_writel(dwc, name, val) \ writel((val), &(__dwc_regs(dwc)->name)) static inline struct dw_dma_chan *to_dw_dma_chan(struct dma_chan *chan) { return container_of(chan, struct dw_dma_chan, chan); } struct dw_dma { struct dma_device dma; char name[20]; void __iomem *regs; struct dma_pool *desc_pool; struct tasklet_struct tasklet; /* channels */ struct dw_dma_chan *chan; u8 all_chan_mask; u8 in_use; /* Channel operations */ void (*initialize_chan)(struct dw_dma_chan *dwc); void (*suspend_chan)(struct dw_dma_chan *dwc, bool drain); void (*resume_chan)(struct dw_dma_chan *dwc, bool drain); u32 (*prepare_ctllo)(struct dw_dma_chan *dwc); void (*encode_maxburst)(struct dw_dma_chan *dwc, u32 *maxburst); u32 (*bytes2block)(struct dw_dma_chan *dwc, size_t bytes, unsigned int width, size_t *len); size_t (*block2bytes)(struct dw_dma_chan *dwc, u32 block, u32 width); /* Device operations */ void (*set_device_name)(struct dw_dma *dw, int id); void (*disable)(struct dw_dma *dw); void (*enable)(struct dw_dma *dw); /* platform data */ struct dw_dma_platform_data *pdata; }; static inline struct dw_dma_regs __iomem *__dw_regs(struct dw_dma *dw) { return dw->regs; } #define dma_readl(dw, name) \ readl(&(__dw_regs(dw)->name)) #define dma_writel(dw, name, val) \ writel((val), &(__dw_regs(dw)->name)) #define idma32_readq(dw, name) \ hi_lo_readq(&(__dw_regs(dw)->name)) #define idma32_writeq(dw, name, val) \ hi_lo_writeq((val), &(__dw_regs(dw)->name)) #define channel_set_bit(dw, reg, mask) \ dma_writel(dw, reg, ((mask) << 8) | (mask)) #define channel_clear_bit(dw, reg, mask) \ dma_writel(dw, reg, ((mask) << 8) | 0) static inline struct dw_dma *to_dw_dma(struct dma_device *ddev) { return container_of(ddev, struct dw_dma, dma); } /* LLI == Linked List Item; a.k.a. DMA block descriptor */ struct dw_lli { /* values that are not changed by hardware */ __le32 sar; __le32 dar; __le32 llp; /* chain to next lli */ __le32 ctllo; /* values that may get written back: */ __le32 ctlhi; /* sstat and dstat can snapshot peripheral register state. * silicon config may discard either or both... */ __le32 sstat; __le32 dstat; }; struct dw_desc { /* FIRST values the hardware uses */ struct dw_lli lli; #define lli_set(d, reg, v) ((d)->lli.reg |= cpu_to_le32(v)) #define lli_clear(d, reg, v) ((d)->lli.reg &= ~cpu_to_le32(v)) #define lli_read(d, reg) le32_to_cpu((d)->lli.reg) #define lli_write(d, reg, v) ((d)->lli.reg = cpu_to_le32(v)) /* THEN values for driver housekeeping */ struct list_head desc_node; struct list_head tx_list; struct dma_async_tx_descriptor txd; size_t len; size_t total_len; u32 residue; }; #define to_dw_desc(h) list_entry(h, struct dw_desc, desc_node) static inline struct dw_desc * txd_to_dw_desc(struct dma_async_tx_descriptor *txd) { return container_of(txd, struct dw_desc, txd); } |