Linux Audio

Check our new training course

Embedded Linux Audio

Check our new training course
with Creative Commons CC-BY-SA
lecture materials

Bootlin logo

Elixir Cross Referencer

Loading...
  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
/* Copyright (C) 2018 Netronome Systems, Inc. */

#include <linux/bitops.h>
#include <linux/kernel.h>
#include <linux/log2.h>

#include "../nfpcore/nfp_cpp.h"
#include "../nfpcore/nfp_nffw.h"
#include "../nfp_app.h"
#include "../nfp_abi.h"
#include "../nfp_main.h"
#include "../nfp_net.h"
#include "main.h"

#define NFP_NUM_PRIOS_SYM_NAME	"_abi_pci_dscp_num_prio_%u"
#define NFP_NUM_BANDS_SYM_NAME	"_abi_pci_dscp_num_band_%u"
#define NFP_ACT_MASK_SYM_NAME	"_abi_nfd_out_q_actions_%u"

#define NFP_RED_SUPPORT_SYM_NAME	"_abi_nfd_out_red_offload_%u"

#define NFP_QLVL_SYM_NAME	"_abi_nfd_out_q_lvls_%u%s"
#define NFP_QLVL_STRIDE		16
#define NFP_QLVL_BLOG_BYTES	0
#define NFP_QLVL_BLOG_PKTS	4
#define NFP_QLVL_THRS		8
#define NFP_QLVL_ACT		12

#define NFP_QMSTAT_SYM_NAME	"_abi_nfdqm%u_stats%s"
#define NFP_QMSTAT_STRIDE	32
#define NFP_QMSTAT_NON_STO	0
#define NFP_QMSTAT_STO		8
#define NFP_QMSTAT_DROP		16
#define NFP_QMSTAT_ECN		24

#define NFP_Q_STAT_SYM_NAME	"_abi_nfd_rxq_stats%u%s"
#define NFP_Q_STAT_STRIDE	16
#define NFP_Q_STAT_PKTS		0
#define NFP_Q_STAT_BYTES	8

#define NFP_NET_ABM_MBOX_CMD		NFP_NET_CFG_MBOX_SIMPLE_CMD
#define NFP_NET_ABM_MBOX_RET		NFP_NET_CFG_MBOX_SIMPLE_RET
#define NFP_NET_ABM_MBOX_DATALEN	NFP_NET_CFG_MBOX_SIMPLE_VAL
#define NFP_NET_ABM_MBOX_RESERVED	(NFP_NET_CFG_MBOX_SIMPLE_VAL + 4)
#define NFP_NET_ABM_MBOX_DATA		(NFP_NET_CFG_MBOX_SIMPLE_VAL + 8)

static int
nfp_abm_ctrl_stat(struct nfp_abm_link *alink, const struct nfp_rtsym *sym,
		  unsigned int stride, unsigned int offset, unsigned int band,
		  unsigned int queue, bool is_u64, u64 *res)
{
	struct nfp_cpp *cpp = alink->abm->app->cpp;
	u64 val, sym_offset;
	unsigned int qid;
	u32 val32;
	int err;

	qid = band * NFP_NET_MAX_RX_RINGS + alink->queue_base + queue;

	sym_offset = qid * stride + offset;
	if (is_u64)
		err = __nfp_rtsym_readq(cpp, sym, 3, 0, sym_offset, &val);
	else
		err = __nfp_rtsym_readl(cpp, sym, 3, 0, sym_offset, &val32);
	if (err) {
		nfp_err(cpp, "RED offload reading stat failed on vNIC %d band %d queue %d (+ %d)\n",
			alink->id, band, queue, alink->queue_base);
		return err;
	}

	*res = is_u64 ? val : val32;
	return 0;
}

int __nfp_abm_ctrl_set_q_lvl(struct nfp_abm *abm, unsigned int id, u32 val)
{
	struct nfp_cpp *cpp = abm->app->cpp;
	u64 sym_offset;
	int err;

	__clear_bit(id, abm->threshold_undef);
	if (abm->thresholds[id] == val)
		return 0;

	sym_offset = id * NFP_QLVL_STRIDE + NFP_QLVL_THRS;
	err = __nfp_rtsym_writel(cpp, abm->q_lvls, 4, 0, sym_offset, val);
	if (err) {
		nfp_err(cpp,
			"RED offload setting level failed on subqueue %d\n",
			id);
		return err;
	}

	abm->thresholds[id] = val;
	return 0;
}

int nfp_abm_ctrl_set_q_lvl(struct nfp_abm_link *alink, unsigned int band,
			   unsigned int queue, u32 val)
{
	unsigned int threshold;

	threshold = band * NFP_NET_MAX_RX_RINGS + alink->queue_base + queue;

	return __nfp_abm_ctrl_set_q_lvl(alink->abm, threshold, val);
}

int __nfp_abm_ctrl_set_q_act(struct nfp_abm *abm, unsigned int id,
			     enum nfp_abm_q_action act)
{
	struct nfp_cpp *cpp = abm->app->cpp;
	u64 sym_offset;
	int err;

	if (abm->actions[id] == act)
		return 0;

	sym_offset = id * NFP_QLVL_STRIDE + NFP_QLVL_ACT;
	err = __nfp_rtsym_writel(cpp, abm->q_lvls, 4, 0, sym_offset, act);
	if (err) {
		nfp_err(cpp,
			"RED offload setting action failed on subqueue %d\n",
			id);
		return err;
	}

	abm->actions[id] = act;
	return 0;
}

int nfp_abm_ctrl_set_q_act(struct nfp_abm_link *alink, unsigned int band,
			   unsigned int queue, enum nfp_abm_q_action act)
{
	unsigned int qid;

	qid = band * NFP_NET_MAX_RX_RINGS + alink->queue_base + queue;

	return __nfp_abm_ctrl_set_q_act(alink->abm, qid, act);
}

u64 nfp_abm_ctrl_stat_non_sto(struct nfp_abm_link *alink, unsigned int queue)
{
	unsigned int band;
	u64 val, sum = 0;

	for (band = 0; band < alink->abm->num_bands; band++) {
		if (nfp_abm_ctrl_stat(alink, alink->abm->qm_stats,
				      NFP_QMSTAT_STRIDE, NFP_QMSTAT_NON_STO,
				      band, queue, true, &val))
			return 0;
		sum += val;
	}

	return sum;
}

u64 nfp_abm_ctrl_stat_sto(struct nfp_abm_link *alink, unsigned int queue)
{
	unsigned int band;
	u64 val, sum = 0;

	for (band = 0; band < alink->abm->num_bands; band++) {
		if (nfp_abm_ctrl_stat(alink, alink->abm->qm_stats,
				      NFP_QMSTAT_STRIDE, NFP_QMSTAT_STO,
				      band, queue, true, &val))
			return 0;
		sum += val;
	}

	return sum;
}

static int
nfp_abm_ctrl_stat_basic(struct nfp_abm_link *alink, unsigned int band,
			unsigned int queue, unsigned int off, u64 *val)
{
	if (!nfp_abm_has_prio(alink->abm)) {
		if (!band) {
			unsigned int id = alink->queue_base + queue;

			*val = nn_readq(alink->vnic,
					NFP_NET_CFG_RXR_STATS(id) + off);
		} else {
			*val = 0;
		}

		return 0;
	} else {
		return nfp_abm_ctrl_stat(alink, alink->abm->q_stats,
					 NFP_Q_STAT_STRIDE, off, band, queue,
					 true, val);
	}
}

int nfp_abm_ctrl_read_q_stats(struct nfp_abm_link *alink, unsigned int band,
			      unsigned int queue, struct nfp_alink_stats *stats)
{
	int err;

	err = nfp_abm_ctrl_stat_basic(alink, band, queue, NFP_Q_STAT_PKTS,
				      &stats->tx_pkts);
	if (err)
		return err;

	err = nfp_abm_ctrl_stat_basic(alink, band, queue, NFP_Q_STAT_BYTES,
				      &stats->tx_bytes);
	if (err)
		return err;

	err = nfp_abm_ctrl_stat(alink, alink->abm->q_lvls, NFP_QLVL_STRIDE,
				NFP_QLVL_BLOG_BYTES, band, queue, false,
				&stats->backlog_bytes);
	if (err)
		return err;

	err = nfp_abm_ctrl_stat(alink, alink->abm->q_lvls,
				NFP_QLVL_STRIDE, NFP_QLVL_BLOG_PKTS,
				band, queue, false, &stats->backlog_pkts);
	if (err)
		return err;

	err = nfp_abm_ctrl_stat(alink, alink->abm->qm_stats,
				NFP_QMSTAT_STRIDE, NFP_QMSTAT_DROP,
				band, queue, true, &stats->drops);
	if (err)
		return err;

	return nfp_abm_ctrl_stat(alink, alink->abm->qm_stats,
				 NFP_QMSTAT_STRIDE, NFP_QMSTAT_ECN,
				 band, queue, true, &stats->overlimits);
}

int nfp_abm_ctrl_read_q_xstats(struct nfp_abm_link *alink,
			       unsigned int band, unsigned int queue,
			       struct nfp_alink_xstats *xstats)
{
	int err;

	err = nfp_abm_ctrl_stat(alink, alink->abm->qm_stats,
				NFP_QMSTAT_STRIDE, NFP_QMSTAT_DROP,
				band, queue, true, &xstats->pdrop);
	if (err)
		return err;

	return nfp_abm_ctrl_stat(alink, alink->abm->qm_stats,
				 NFP_QMSTAT_STRIDE, NFP_QMSTAT_ECN,
				 band, queue, true, &xstats->ecn_marked);
}

int nfp_abm_ctrl_qm_enable(struct nfp_abm *abm)
{
	return nfp_mbox_cmd(abm->app->pf, NFP_MBOX_PCIE_ABM_ENABLE,
			    NULL, 0, NULL, 0);
}

int nfp_abm_ctrl_qm_disable(struct nfp_abm *abm)
{
	return nfp_mbox_cmd(abm->app->pf, NFP_MBOX_PCIE_ABM_DISABLE,
			    NULL, 0, NULL, 0);
}

int nfp_abm_ctrl_prio_map_update(struct nfp_abm_link *alink, u32 *packed)
{
	struct nfp_net *nn = alink->vnic;
	unsigned int i;
	int err;

	/* Write data_len and wipe reserved */
	nn_writeq(nn, nn->tlv_caps.mbox_off + NFP_NET_ABM_MBOX_DATALEN,
		  alink->abm->prio_map_len);

	for (i = 0; i < alink->abm->prio_map_len; i += sizeof(u32))
		nn_writel(nn, nn->tlv_caps.mbox_off + NFP_NET_ABM_MBOX_DATA + i,
			  packed[i / sizeof(u32)]);

	err = nfp_net_reconfig_mbox(nn,
				    NFP_NET_CFG_MBOX_CMD_PCI_DSCP_PRIOMAP_SET);
	if (err)
		nfp_err(alink->abm->app->cpp,
			"setting DSCP -> VQ map failed with error %d\n", err);
	return err;
}

static int nfp_abm_ctrl_prio_check_params(struct nfp_abm_link *alink)
{
	struct nfp_abm *abm = alink->abm;
	struct nfp_net *nn = alink->vnic;
	unsigned int min_mbox_sz;

	if (!nfp_abm_has_prio(alink->abm))
		return 0;

	min_mbox_sz = NFP_NET_ABM_MBOX_DATA + alink->abm->prio_map_len;
	if (nn->tlv_caps.mbox_len < min_mbox_sz) {
		nfp_err(abm->app->pf->cpp, "vNIC mailbox too small for prio offload: %u, need: %u\n",
			nn->tlv_caps.mbox_len,  min_mbox_sz);
		return -EINVAL;
	}

	return 0;
}

int nfp_abm_ctrl_read_params(struct nfp_abm_link *alink)
{
	alink->queue_base = nn_readl(alink->vnic, NFP_NET_CFG_START_RXQ);
	alink->queue_base /= alink->vnic->stride_rx;

	return nfp_abm_ctrl_prio_check_params(alink);
}

static unsigned int nfp_abm_ctrl_prio_map_size(struct nfp_abm *abm)
{
	unsigned int size;

	size = roundup_pow_of_two(order_base_2(abm->num_bands));
	size = DIV_ROUND_UP(size * abm->num_prios, BITS_PER_BYTE);
	size = round_up(size, sizeof(u32));

	return size;
}

static const struct nfp_rtsym *
nfp_abm_ctrl_find_rtsym(struct nfp_pf *pf, const char *name, unsigned int size)
{
	const struct nfp_rtsym *sym;

	sym = nfp_rtsym_lookup(pf->rtbl, name);
	if (!sym) {
		nfp_err(pf->cpp, "Symbol '%s' not found\n", name);
		return ERR_PTR(-ENOENT);
	}
	if (nfp_rtsym_size(sym) != size) {
		nfp_err(pf->cpp,
			"Symbol '%s' wrong size: expected %u got %llu\n",
			name, size, nfp_rtsym_size(sym));
		return ERR_PTR(-EINVAL);
	}

	return sym;
}

static const struct nfp_rtsym *
nfp_abm_ctrl_find_q_rtsym(struct nfp_abm *abm, const char *name_fmt,
			  size_t size)
{
	char pf_symbol[64];

	size = array3_size(size, abm->num_bands, NFP_NET_MAX_RX_RINGS);
	snprintf(pf_symbol, sizeof(pf_symbol), name_fmt,
		 abm->pf_id, nfp_abm_has_prio(abm) ? "_per_band" : "");

	return nfp_abm_ctrl_find_rtsym(abm->app->pf, pf_symbol, size);
}

int nfp_abm_ctrl_find_addrs(struct nfp_abm *abm)
{
	struct nfp_pf *pf = abm->app->pf;
	const struct nfp_rtsym *sym;
	int res;

	abm->pf_id = nfp_cppcore_pcie_unit(pf->cpp);

	/* Check if Qdisc offloads are supported */
	res = nfp_pf_rtsym_read_optional(pf, NFP_RED_SUPPORT_SYM_NAME, 1);
	if (res < 0)
		return res;
	abm->red_support = res;

	/* Read count of prios and prio bands */
	res = nfp_pf_rtsym_read_optional(pf, NFP_NUM_BANDS_SYM_NAME, 1);
	if (res < 0)
		return res;
	abm->num_bands = res;

	res = nfp_pf_rtsym_read_optional(pf, NFP_NUM_PRIOS_SYM_NAME, 1);
	if (res < 0)
		return res;
	abm->num_prios = res;

	/* Read available actions */
	res = nfp_pf_rtsym_read_optional(pf, NFP_ACT_MASK_SYM_NAME,
					 BIT(NFP_ABM_ACT_MARK_DROP));
	if (res < 0)
		return res;
	abm->action_mask = res;

	abm->prio_map_len = nfp_abm_ctrl_prio_map_size(abm);
	abm->dscp_mask = GENMASK(7, 8 - order_base_2(abm->num_prios));

	/* Check values are sane, U16_MAX is arbitrarily chosen as max */
	if (!is_power_of_2(abm->num_bands) || !is_power_of_2(abm->num_prios) ||
	    abm->num_bands > U16_MAX || abm->num_prios > U16_MAX ||
	    (abm->num_bands == 1) != (abm->num_prios == 1)) {
		nfp_err(pf->cpp,
			"invalid priomap description num bands: %u and num prios: %u\n",
			abm->num_bands, abm->num_prios);
		return -EINVAL;
	}

	/* Find level and stat symbols */
	if (!abm->red_support)
		return 0;

	sym = nfp_abm_ctrl_find_q_rtsym(abm, NFP_QLVL_SYM_NAME,
					NFP_QLVL_STRIDE);
	if (IS_ERR(sym))
		return PTR_ERR(sym);
	abm->q_lvls = sym;

	sym = nfp_abm_ctrl_find_q_rtsym(abm, NFP_QMSTAT_SYM_NAME,
					NFP_QMSTAT_STRIDE);
	if (IS_ERR(sym))
		return PTR_ERR(sym);
	abm->qm_stats = sym;

	if (nfp_abm_has_prio(abm)) {
		sym = nfp_abm_ctrl_find_q_rtsym(abm, NFP_Q_STAT_SYM_NAME,
						NFP_Q_STAT_STRIDE);
		if (IS_ERR(sym))
			return PTR_ERR(sym);
		abm->q_stats = sym;
	}

	return 0;
}