Loading...
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 | /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ /* * Copyright(c) 2015-2017 Intel Corporation. */ #ifndef _PIO_H #define _PIO_H /* send context types */ #define SC_KERNEL 0 #define SC_VL15 1 #define SC_ACK 2 #define SC_USER 3 /* must be the last one: it may take all left */ #define SC_MAX 4 /* count of send context types */ /* invalid send context index */ #define INVALID_SCI 0xff /* PIO buffer release callback function */ typedef void (*pio_release_cb)(void *arg, int code); /* PIO release codes - in bits, as there could more than one that apply */ #define PRC_OK 0 /* no known error */ #define PRC_STATUS_ERR 0x01 /* credit return due to status error */ #define PRC_PBC 0x02 /* credit return due to PBC */ #define PRC_THRESHOLD 0x04 /* credit return due to threshold */ #define PRC_FILL_ERR 0x08 /* credit return due fill error */ #define PRC_FORCE 0x10 /* credit return due credit force */ #define PRC_SC_DISABLE 0x20 /* clean-up after a context disable */ /* byte helper */ union mix { u64 val64; u32 val32[2]; u8 val8[8]; }; /* an allocated PIO buffer */ struct pio_buf { struct send_context *sc;/* back pointer to owning send context */ pio_release_cb cb; /* called when the buffer is released */ void *arg; /* argument for cb */ void __iomem *start; /* buffer start address */ void __iomem *end; /* context end address */ unsigned long sent_at; /* buffer is sent when <= free */ union mix carry; /* pending unwritten bytes */ u16 qw_written; /* QW written so far */ u8 carry_bytes; /* number of valid bytes in carry */ }; /* cache line aligned pio buffer array */ union pio_shadow_ring { struct pio_buf pbuf; } ____cacheline_aligned; /* per-NUMA send context */ struct send_context { /* read-only after init */ struct hfi1_devdata *dd; /* device */ union pio_shadow_ring *sr; /* shadow ring */ void __iomem *base_addr; /* start of PIO memory */ u32 __percpu *buffers_allocated;/* count of buffers allocated */ u32 size; /* context size, in bytes */ int node; /* context home node */ u32 sr_size; /* size of the shadow ring */ u16 flags; /* flags */ u8 type; /* context type */ u8 sw_index; /* software index number */ u8 hw_context; /* hardware context number */ u8 group; /* credit return group */ /* allocator fields */ spinlock_t alloc_lock ____cacheline_aligned_in_smp; u32 sr_head; /* shadow ring head */ unsigned long fill; /* official alloc count */ unsigned long alloc_free; /* copy of free (less cache thrash) */ u32 fill_wrap; /* tracks fill within ring */ u32 credits; /* number of blocks in context */ /* adding a new field here would make it part of this cacheline */ /* releaser fields */ spinlock_t release_lock ____cacheline_aligned_in_smp; u32 sr_tail; /* shadow ring tail */ unsigned long free; /* official free count */ volatile __le64 *hw_free; /* HW free counter */ /* list for PIO waiters */ struct list_head piowait ____cacheline_aligned_in_smp; seqlock_t waitlock; spinlock_t credit_ctrl_lock ____cacheline_aligned_in_smp; u32 credit_intr_count; /* count of credit intr users */ u64 credit_ctrl; /* cache for credit control */ wait_queue_head_t halt_wait; /* wait until kernel sees interrupt */ struct work_struct halt_work; /* halted context work queue entry */ }; /* send context flags */ #define SCF_ENABLED 0x01 #define SCF_IN_FREE 0x02 #define SCF_HALTED 0x04 #define SCF_FROZEN 0x08 #define SCF_LINK_DOWN 0x10 struct send_context_info { struct send_context *sc; /* allocated working context */ u16 allocated; /* has this been allocated? */ u16 type; /* context type */ u16 base; /* base in PIO array */ u16 credits; /* size in PIO array */ }; /* DMA credit return, index is always (context & 0x7) */ struct credit_return { volatile __le64 cr[8]; }; /* NUMA indexed credit return array */ struct credit_return_base { struct credit_return *va; dma_addr_t dma; }; /* send context configuration sizes (one per type) */ struct sc_config_sizes { short int size; short int count; }; /* * The diagram below details the relationship of the mapping structures * * Since the mapping now allows for non-uniform send contexts per vl, the * number of send contexts for a vl is either the vl_scontexts[vl] or * a computation based on num_kernel_send_contexts/num_vls: * * For example: * nactual = vl_scontexts ? vl_scontexts[vl] : num_kernel_send_contexts/num_vls * * n = roundup to next highest power of 2 using nactual * * In the case where there are num_kernel_send_contexts/num_vls doesn't divide * evenly, the extras are added from the last vl downward. * * For the case where n > nactual, the send contexts are assigned * in a round robin fashion wrapping back to the first send context * for a particular vl. * * dd->pio_map * | pio_map_elem[0] * | +--------------------+ * v | mask | * pio_vl_map |--------------------| * +--------------------------+ | ksc[0] -> sc 1 | * | list (RCU) | |--------------------| * |--------------------------| ->| ksc[1] -> sc 2 | * | mask | --/ |--------------------| * |--------------------------| -/ | * | * | actual_vls (max 8) | -/ |--------------------| * |--------------------------| --/ | ksc[n-1] -> sc n | * | vls (max 8) | -/ +--------------------+ * |--------------------------| --/ * | map[0] |-/ * |--------------------------| +--------------------+ * | map[1] |--- | mask | * |--------------------------| \---- |--------------------| * | * | \-- | ksc[0] -> sc 1+n | * | * | \---- |--------------------| * | * | \->| ksc[1] -> sc 2+n | * |--------------------------| |--------------------| * | map[vls - 1] |- | * | * +--------------------------+ \- |--------------------| * \- | ksc[m-1] -> sc m+n | * \ +--------------------+ * \- * \ * \- +----------------------+ * \- | mask | * \ |----------------------| * \- | ksc[0] -> sc 1+m+n | * \- |----------------------| * >| ksc[1] -> sc 2+m+n | * |----------------------| * | * | * |----------------------| * | ksc[o-1] -> sc o+m+n | * +----------------------+ * */ /* Initial number of send contexts per VL */ #define INIT_SC_PER_VL 2 /* * struct pio_map_elem - mapping for a vl * @mask - selector mask * @ksc - array of kernel send contexts for this vl * * The mask is used to "mod" the selector to * produce index into the trailing array of * kscs */ struct pio_map_elem { u32 mask; struct send_context *ksc[]; }; /* * struct pio_vl_map - mapping for a vl * @list - rcu head for free callback * @mask - vl mask to "mod" the vl to produce an index to map array * @actual_vls - number of vls * @vls - numbers of vls rounded to next power of 2 * @map - array of pio_map_elem entries * * This is the parent mapping structure. The trailing members of the * struct point to pio_map_elem entries, which in turn point to an * array of kscs for that vl. */ struct pio_vl_map { struct rcu_head list; u32 mask; u8 actual_vls; u8 vls; struct pio_map_elem *map[]; }; int pio_map_init(struct hfi1_devdata *dd, u8 port, u8 num_vls, u8 *vl_scontexts); void free_pio_map(struct hfi1_devdata *dd); struct send_context *pio_select_send_context_vl(struct hfi1_devdata *dd, u32 selector, u8 vl); struct send_context *pio_select_send_context_sc(struct hfi1_devdata *dd, u32 selector, u8 sc5); /* send context functions */ int init_credit_return(struct hfi1_devdata *dd); void free_credit_return(struct hfi1_devdata *dd); int init_sc_pools_and_sizes(struct hfi1_devdata *dd); int init_send_contexts(struct hfi1_devdata *dd); int init_pervl_scs(struct hfi1_devdata *dd); struct send_context *sc_alloc(struct hfi1_devdata *dd, int type, uint hdrqentsize, int numa); void sc_free(struct send_context *sc); int sc_enable(struct send_context *sc); void sc_disable(struct send_context *sc); int sc_restart(struct send_context *sc); void sc_return_credits(struct send_context *sc); void sc_flush(struct send_context *sc); void sc_drop(struct send_context *sc); void sc_stop(struct send_context *sc, int bit); struct pio_buf *sc_buffer_alloc(struct send_context *sc, u32 dw_len, pio_release_cb cb, void *arg); void sc_release_update(struct send_context *sc); void sc_group_release_update(struct hfi1_devdata *dd, u32 hw_context); void sc_add_credit_return_intr(struct send_context *sc); void sc_del_credit_return_intr(struct send_context *sc); void sc_set_cr_threshold(struct send_context *sc, u32 new_threshold); u32 sc_percent_to_threshold(struct send_context *sc, u32 percent); u32 sc_mtu_to_threshold(struct send_context *sc, u32 mtu, u32 hdrqentsize); void hfi1_sc_wantpiobuf_intr(struct send_context *sc, u32 needint); void sc_wait(struct hfi1_devdata *dd); void set_pio_integrity(struct send_context *sc); /* support functions */ void pio_reset_all(struct hfi1_devdata *dd); void pio_freeze(struct hfi1_devdata *dd); void pio_kernel_unfreeze(struct hfi1_devdata *dd); void pio_kernel_linkup(struct hfi1_devdata *dd); /* global PIO send control operations */ #define PSC_GLOBAL_ENABLE 0 #define PSC_GLOBAL_DISABLE 1 #define PSC_GLOBAL_VLARB_ENABLE 2 #define PSC_GLOBAL_VLARB_DISABLE 3 #define PSC_CM_RESET 4 #define PSC_DATA_VL_ENABLE 5 #define PSC_DATA_VL_DISABLE 6 void __cm_reset(struct hfi1_devdata *dd, u64 sendctrl); void pio_send_control(struct hfi1_devdata *dd, int op); /* PIO copy routines */ void pio_copy(struct hfi1_devdata *dd, struct pio_buf *pbuf, u64 pbc, const void *from, size_t count); void seg_pio_copy_start(struct pio_buf *pbuf, u64 pbc, const void *from, size_t nbytes); void seg_pio_copy_mid(struct pio_buf *pbuf, const void *from, size_t nbytes); void seg_pio_copy_end(struct pio_buf *pbuf); void seqfile_dump_sci(struct seq_file *s, u32 i, struct send_context_info *sci); #endif /* _PIO_H */ |