diff options
Diffstat (limited to 'net/sched')
36 files changed, 478 insertions, 232 deletions
diff --git a/net/sched/act_api.c b/net/sched/act_api.c index 04ea11c90e03..b4a7e1c63cb8 100644 --- a/net/sched/act_api.c +++ b/net/sched/act_api.c @@ -148,10 +148,15 @@ static void offload_action_hw_count_dec(struct tc_action *act, static unsigned int tcf_offload_act_num_actions_single(struct tc_action *act) { - if (is_tcf_pedit(act)) - return tcf_pedit_nkeys(act); - else - return 1; + unsigned int count; + + if (is_tcf_pedit(act)) { + spin_lock_bh(&act->tcfa_lock); + count = tcf_pedit_nkeys_locked(act); + spin_unlock_bh(&act->tcfa_lock); + return count; + } + return 1; } static bool tc_act_skip_hw(u32 flags) diff --git a/net/sched/act_bpf.c b/net/sched/act_bpf.c index c2b5bc19e091..284800b7d3c5 100644 --- a/net/sched/act_bpf.c +++ b/net/sched/act_bpf.c @@ -44,7 +44,7 @@ TC_INDIRECT_SCOPE int tcf_bpf_act(struct sk_buff *skb, tcf_lastuse_update(&prog->tcf_tm); bstats_update(this_cpu_ptr(prog->common.cpu_bstats), skb); - filter = rcu_dereference(prog->filter); + filter = rcu_dereference_bh(prog->filter); if (at_ingress) { __skb_push(skb, skb->mac_len); filter_res = bpf_prog_run_data_pointers(filter, skb); diff --git a/net/sched/act_ct.c b/net/sched/act_ct.c index 6158e13c98d3..be535a261fa0 100644 --- a/net/sched/act_ct.c +++ b/net/sched/act_ct.c @@ -844,11 +844,11 @@ static int tcf_ct_handle_fragments(struct net *net, struct sk_buff *skb, u8 family, u16 zone, bool *defrag) { enum ip_conntrack_info ctinfo; + struct tc_skb_cb cb; struct nf_conn *ct; int err = 0; bool frag; u8 proto; - u16 mru; /* Previously seen (loopback)? Ignore. */ ct = nf_ct_get(skb, &ctinfo); @@ -862,12 +862,13 @@ static int tcf_ct_handle_fragments(struct net *net, struct sk_buff *skb, if (err || !frag) return err; - err = nf_ct_handle_fragments(net, skb, zone, family, &proto, &mru); + cb = *tc_skb_cb(skb); + err = nf_ct_handle_fragments(net, skb, zone, family, &proto, &cb.mru); if (err) return err; *defrag = true; - tc_skb_cb(skb)->mru = mru; + *tc_skb_cb(skb) = cb; return 0; } @@ -1295,7 +1296,8 @@ static int tcf_ct_fill_params(struct net *net, if (tb[TCA_CT_ZONE]) { if (!IS_ENABLED(CONFIG_NF_CONNTRACK_ZONES)) { NL_SET_ERR_MSG_MOD(extack, "Conntrack zones isn't enabled."); - return -EOPNOTSUPP; + err = -EOPNOTSUPP; + goto err; } tcf_ct_set_key_val(tb, @@ -1308,7 +1310,8 @@ static int tcf_ct_fill_params(struct net *net, tmpl = nf_ct_tmpl_alloc(net, &zone, GFP_KERNEL); if (!tmpl) { NL_SET_ERR_MSG_MOD(extack, "Failed to allocate conntrack template"); - return -ENOMEM; + err = -ENOMEM; + goto err; } p->tmpl = tmpl; if (tb[TCA_CT_HELPER_NAME]) { diff --git a/net/sched/act_pedit.c b/net/sched/act_pedit.c index bd3b1da3cd63..437a0563786b 100644 --- a/net/sched/act_pedit.c +++ b/net/sched/act_pedit.c @@ -563,9 +563,18 @@ static int tcf_pedit_offload_act_setup(struct tc_action *act, void *entry_data, { if (bind) { struct flow_action_entry *entry = entry_data; + int nkeys = tcf_pedit_nkeys_locked(act); int k; - for (k = 0; k < tcf_pedit_nkeys(act); k++) { + /* If the required keys exceed the remaining capacity return + * -ENOSPC to abort the offload and fallback to software. + */ + if (nkeys > *index_inc) { + NL_SET_ERR_MSG_MOD(extack, "Not enough space to offload all pedit keys"); + return -ENOSPC; + } + + for (k = 0; k < nkeys; k++) { switch (tcf_pedit_cmd(act, k)) { case TCA_PEDIT_KEY_EX_CMD_SET: entry->id = FLOW_ACTION_MANGLE; @@ -602,7 +611,7 @@ static int tcf_pedit_offload_act_setup(struct tc_action *act, void *entry_data, return -EOPNOTSUPP; } - for (k = 1; k < tcf_pedit_nkeys(act); k++) { + for (k = 1; k < tcf_pedit_nkeys_locked(act); k++) { if (cmd != tcf_pedit_cmd(act, k)) { NL_SET_ERR_MSG_MOD(extack, "Unsupported pedit command offload"); return -EOPNOTSUPP; diff --git a/net/sched/cls_api.c b/net/sched/cls_api.c index 20f7f9ee0b35..ffeea6db8337 100644 --- a/net/sched/cls_api.c +++ b/net/sched/cls_api.c @@ -3886,12 +3886,21 @@ int tc_setup_action(struct flow_action *flow_action, entry = &flow_action->entries[j]; spin_lock_bh(&act->tcfa_lock); + + /* Abort the offload if we have exhausted the allocated capacity */ + if (j >= flow_action->num_entries) { + NL_SET_ERR_MSG_MOD(extack, "Flow action buffer overflow"); + err = -ENOSPC; + goto err_out_locked; + } + err = tcf_act_get_user_cookie(entry, act); if (err) goto err_out_locked; - index = 0; - err = tc_setup_offload_act(act, entry, &index, extack); + index = flow_action->num_entries - j; + err = tc_setup_offload_act(act, entry, &index, + extack); if (err) goto err_out_locked; @@ -3945,10 +3954,13 @@ unsigned int tcf_exts_num_actions(struct tcf_exts *exts) int i; tcf_exts_for_each_action(i, act, exts) { - if (is_tcf_pedit(act)) - num_acts += tcf_pedit_nkeys(act); - else + if (is_tcf_pedit(act)) { + spin_lock_bh(&act->tcfa_lock); + num_acts += tcf_pedit_nkeys_locked(act); + spin_unlock_bh(&act->tcfa_lock); + } else { num_acts++; + } } return num_acts; } @@ -4049,6 +4061,9 @@ struct sk_buff *tcf_qevent_handle(struct tcf_qevent *qe, struct Qdisc *sch, stru skb_do_redirect(skb); *ret = __NET_XMIT_STOLEN; return NULL; + case TC_ACT_CONSUMED: + *ret = __NET_XMIT_STOLEN; + return NULL; } return skb; diff --git a/net/sched/cls_bpf.c b/net/sched/cls_bpf.c index 9a346b6221b3..001d8c4ebfed 100644 --- a/net/sched/cls_bpf.c +++ b/net/sched/cls_bpf.c @@ -142,7 +142,8 @@ static bool cls_bpf_is_ebpf(const struct cls_bpf_prog *prog) static int cls_bpf_offload_cmd(struct tcf_proto *tp, struct cls_bpf_prog *prog, struct cls_bpf_prog *oldprog, - struct netlink_ext_ack *extack) + struct netlink_ext_ack *extack, + bool is_rollback) { struct tcf_block *block = tp->chain->block; struct tc_cls_bpf_offload cls_bpf = {}; @@ -177,7 +178,8 @@ static int cls_bpf_offload_cmd(struct tcf_proto *tp, struct cls_bpf_prog *prog, &oldprog->in_hw_count, true); if (prog && err) { - cls_bpf_offload_cmd(tp, oldprog, prog, extack); + if (!is_rollback) + cls_bpf_offload_cmd(tp, oldprog, prog, extack, true); return err; } @@ -208,7 +210,7 @@ static int cls_bpf_offload(struct tcf_proto *tp, struct cls_bpf_prog *prog, if (!prog && !oldprog) return 0; - return cls_bpf_offload_cmd(tp, prog, oldprog, extack); + return cls_bpf_offload_cmd(tp, prog, oldprog, extack, false); } static void cls_bpf_stop_offload(struct tcf_proto *tp, @@ -217,7 +219,7 @@ static void cls_bpf_stop_offload(struct tcf_proto *tp, { int err; - err = cls_bpf_offload_cmd(tp, NULL, prog, extack); + err = cls_bpf_offload_cmd(tp, NULL, prog, extack, false); if (err) pr_err("Stopping hardware offload failed: %d\n", err); } diff --git a/net/sched/cls_flow.c b/net/sched/cls_flow.c index ab364e4e4686..356c68ebc389 100644 --- a/net/sched/cls_flow.c +++ b/net/sched/cls_flow.c @@ -21,6 +21,7 @@ #include <net/inet_sock.h> #include <net/pkt_cls.h> +#include <linux/siphash.h> #include <net/ip.h> #include <net/route.h> #include <net/flow_dissector.h> @@ -57,11 +58,15 @@ struct flow_filter { struct rcu_work rwork; }; +static siphash_aligned_key_t flow_keys_secret __read_mostly; + static inline u32 addr_fold(void *addr) { - unsigned long a = (unsigned long)addr; - - return (a & 0xFFFFFFFF) ^ (BITS_PER_LONG > 32 ? a >> 32 : 0); +#ifdef CONFIG_64BIT + return (u32)siphash_1u64((u64)addr, &flow_keys_secret); +#else + return (u32)siphash_1u32((u32)addr, &flow_keys_secret); +#endif } static u32 flow_get_src(const struct sk_buff *skb, const struct flow_keys *flow) @@ -596,6 +601,7 @@ static int flow_init(struct tcf_proto *tp) return -ENOBUFS; INIT_LIST_HEAD(&head->filters); rcu_assign_pointer(tp->root, head); + net_get_random_once(&flow_keys_secret, sizeof(flow_keys_secret)); return 0; } diff --git a/net/sched/sch_api.c b/net/sched/sch_api.c index ed869a5ffc73..0dd3efd86393 100644 --- a/net/sched/sch_api.c +++ b/net/sched/sch_api.c @@ -805,7 +805,7 @@ void qdisc_tree_reduce_backlog(struct Qdisc *sch, int n, int len) cl = cops->find(sch, parentid); cops->qlen_notify(sch, cl); } - sch->q.qlen -= n; + WRITE_ONCE(sch->q.qlen, sch->q.qlen - n); sch->qstats.backlog -= len; __qdisc_qstats_drop(sch, drops); } diff --git a/net/sched/sch_cake.c b/net/sched/sch_cake.c index 5862933be8d7..07ea175657bb 100644 --- a/net/sched/sch_cake.c +++ b/net/sched/sch_cake.c @@ -1389,10 +1389,7 @@ static u32 cake_calc_overhead(struct cake_sched_data *qd, u32 len, u32 off) if (qd->min_netlen > len) WRITE_ONCE(qd->min_netlen, len); - len += q->rate_overhead; - - if (len < q->rate_mpu) - len = q->rate_mpu; + len = max((s32)len + q->rate_overhead, (s32)q->rate_mpu); if (q->atm_mode == CAKE_ATM_ATM) { len += 47; @@ -1612,7 +1609,7 @@ static unsigned int cake_drop(struct Qdisc *sch, struct sk_buff **to_free) cake_advance_shaper(q, b, skb, now, true); qdisc_drop_reason(skb, sch, to_free, QDISC_DROP_OVERLIMIT); - sch->q.qlen--; + qdisc_qlen_dec(sch); cake_heapify(q, 0); @@ -1822,7 +1819,7 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch, segs); flow_queue_add(flow, segs); - sch->q.qlen++; + qdisc_qlen_inc(sch); numsegs++; slen += segs->len; q->buffer_used += segs->truesize; @@ -1861,7 +1858,7 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch, qdisc_tree_reduce_backlog(sch, 1, ack_pkt_len); consume_skb(ack); } else { - sch->q.qlen++; + qdisc_qlen_inc(sch); q->buffer_used += skb->truesize; } @@ -1987,7 +1984,7 @@ static struct sk_buff *cake_dequeue_one(struct Qdisc *sch) WRITE_ONCE(b->tin_backlog, b->tin_backlog - len); sch->qstats.backlog -= len; q->buffer_used -= skb->truesize; - sch->q.qlen--; + qdisc_qlen_dec(sch); if (q->overflow_timeout) cake_heapify(q, b->overflow_idx[q->cur_flow]); diff --git a/net/sched/sch_cbs.c b/net/sched/sch_cbs.c index 0f953bd46b58..0994da70816e 100644 --- a/net/sched/sch_cbs.c +++ b/net/sched/sch_cbs.c @@ -97,7 +97,7 @@ static int cbs_child_enqueue(struct sk_buff *skb, struct Qdisc *sch, return err; sch->qstats.backlog += len; - sch->q.qlen++; + qdisc_qlen_inc(sch); return NET_XMIT_SUCCESS; } @@ -168,7 +168,7 @@ static struct sk_buff *cbs_child_dequeue(struct Qdisc *sch, struct Qdisc *child) qdisc_qstats_backlog_dec(sch, skb); qdisc_bstats_update(sch, skb); - sch->q.qlen--; + qdisc_qlen_dec(sch); return skb; } diff --git a/net/sched/sch_choke.c b/net/sched/sch_choke.c index 2875bcdb18a4..73d3e673dc7b 100644 --- a/net/sched/sch_choke.c +++ b/net/sched/sch_choke.c @@ -123,7 +123,7 @@ static void choke_drop_by_idx(struct Qdisc *sch, unsigned int idx, if (idx == q->tail) choke_zap_tail_holes(q); - --sch->q.qlen; + qdisc_qlen_dec(sch); qdisc_qstats_backlog_dec(sch, skb); qdisc_tree_reduce_backlog(sch, 1, qdisc_pkt_len(skb)); qdisc_drop(skb, sch, to_free); @@ -271,7 +271,7 @@ static int choke_enqueue(struct sk_buff *skb, struct Qdisc *sch, if (sch->q.qlen < q->limit) { q->tab[q->tail] = skb; q->tail = (q->tail + 1) & q->tab_mask; - ++sch->q.qlen; + qdisc_qlen_inc(sch); qdisc_qstats_backlog_inc(sch, skb); return NET_XMIT_SUCCESS; } @@ -298,7 +298,7 @@ static struct sk_buff *choke_dequeue(struct Qdisc *sch) skb = q->tab[q->head]; q->tab[q->head] = NULL; choke_zap_head_holes(q); - --sch->q.qlen; + qdisc_qlen_dec(sch); qdisc_qstats_backlog_dec(sch, skb); qdisc_bstats_update(sch, skb); @@ -396,7 +396,7 @@ static int choke_change(struct Qdisc *sch, struct nlattr *opt, } dropped += qdisc_pkt_len(skb); qdisc_qstats_backlog_dec(sch, skb); - --sch->q.qlen; + qdisc_qlen_dec(sch); rtnl_qdisc_drop(skb, sch); } qdisc_tree_reduce_backlog(sch, oqlen - sch->q.qlen, dropped); diff --git a/net/sched/sch_codel.c b/net/sched/sch_codel.c index 317aae0ec7bd..7d5076196aff 100644 --- a/net/sched/sch_codel.c +++ b/net/sched/sch_codel.c @@ -56,7 +56,7 @@ static void drop_func(struct sk_buff *skb, void *ctx) qdisc_qstats_drop(sch); } -static struct sk_buff *codel_qdisc_dequeue(struct Qdisc *sch) +static struct sk_buff *__codel_qdisc_dequeue(struct Qdisc *sch) { struct codel_sched_data *q = qdisc_priv(sch); struct sk_buff *skb; @@ -65,13 +65,51 @@ static struct sk_buff *codel_qdisc_dequeue(struct Qdisc *sch) &q->stats, qdisc_pkt_len, codel_get_enqueue_time, drop_func, dequeue_func); + if (skb) + qdisc_bstats_update(sch, skb); + return skb; +} + +static void codel_dequeue_drop(struct Qdisc *sch) +{ + struct codel_sched_data *q = qdisc_priv(sch); + if (q->stats.drop_count) { - qdisc_tree_reduce_backlog(sch, q->stats.drop_count, q->stats.drop_len); + qdisc_tree_reduce_backlog(sch, q->stats.drop_count, + q->stats.drop_len); q->stats.drop_count = 0; q->stats.drop_len = 0; } - if (skb) - qdisc_bstats_update(sch, skb); +} + +static struct sk_buff *codel_qdisc_dequeue(struct Qdisc *sch) +{ + struct sk_buff *skb; + + skb = __codel_qdisc_dequeue(sch); + + codel_dequeue_drop(sch); + + return skb; +} + +static struct sk_buff *codel_peek(struct Qdisc *sch) +{ + struct sk_buff *skb = skb_peek(&sch->gso_skb); + + if (!skb) { + skb = __codel_qdisc_dequeue(sch); + + if (skb) { + __skb_queue_head(&sch->gso_skb, skb); + /* it's still part of the queue */ + qdisc_qstats_backlog_inc(sch, skb); + sch->q.qlen++; + } + + codel_dequeue_drop(sch); + } + return skb; } @@ -257,7 +295,7 @@ static struct Qdisc_ops codel_qdisc_ops __read_mostly = { .enqueue = codel_qdisc_enqueue, .dequeue = codel_qdisc_dequeue, - .peek = qdisc_peek_dequeued, + .peek = codel_peek, .init = codel_init, .reset = codel_reset, .change = codel_change, diff --git a/net/sched/sch_drr.c b/net/sched/sch_drr.c index 01335a49e091..2eedd3a4322a 100644 --- a/net/sched/sch_drr.c +++ b/net/sched/sch_drr.c @@ -270,7 +270,7 @@ static int drr_dump_class_stats(struct Qdisc *sch, unsigned long arg, memset(&xstats, 0, sizeof(xstats)); if (qlen) - xstats.deficit = cl->deficit; + xstats.deficit = READ_ONCE(cl->deficit); if (gnet_stats_copy_basic(d, NULL, &cl->bstats, true) < 0 || gnet_stats_copy_rate_est(d, &cl->rate_est) < 0 || @@ -362,11 +362,11 @@ static int drr_enqueue(struct sk_buff *skb, struct Qdisc *sch, if (!cl_is_active(cl)) { list_add_tail(&cl->alist, &q->active); - cl->deficit = cl->quantum; + WRITE_ONCE(cl->deficit, cl->quantum); } sch->qstats.backlog += len; - sch->q.qlen++; + qdisc_qlen_inc(sch); return err; } @@ -389,7 +389,7 @@ static struct sk_buff *drr_dequeue(struct Qdisc *sch) len = qdisc_pkt_len(skb); if (len <= cl->deficit) { - cl->deficit -= len; + WRITE_ONCE(cl->deficit, cl->deficit - len); skb = qdisc_dequeue_peeked(cl->qdisc); if (unlikely(skb == NULL)) goto out; @@ -399,11 +399,11 @@ static struct sk_buff *drr_dequeue(struct Qdisc *sch) bstats_update(&cl->bstats, skb); qdisc_bstats_update(sch, skb); qdisc_qstats_backlog_dec(sch, skb); - sch->q.qlen--; + qdisc_qlen_dec(sch); return skb; } - cl->deficit += cl->quantum; + WRITE_ONCE(cl->deficit, cl->deficit + cl->quantum); list_move_tail(&cl->alist, &q->active); } out: diff --git a/net/sched/sch_dualpi2.c b/net/sched/sch_dualpi2.c index 1ca4d2c03e61..27088760eff4 100644 --- a/net/sched/sch_dualpi2.c +++ b/net/sched/sch_dualpi2.c @@ -190,7 +190,7 @@ static bool skb_apply_step(struct sk_buff *skb, struct dualpi2_sched_data *q) static bool dualpi2_mark(struct dualpi2_sched_data *q, struct sk_buff *skb) { if (INET_ECN_set_ce(skb)) { - q->ecn_mark++; + WRITE_ONCE(q->ecn_mark, q->ecn_mark + 1); return true; } return false; @@ -198,7 +198,7 @@ static bool dualpi2_mark(struct dualpi2_sched_data *q, struct sk_buff *skb) static void dualpi2_reset_c_protection(struct dualpi2_sched_data *q) { - q->c_protection_credit = q->c_protection_init; + WRITE_ONCE(q->c_protection_credit, q->c_protection_init); } /* This computes the initial credit value and WRR weight for the L queue (wl) @@ -346,6 +346,8 @@ static int dualpi2_skb_classify(struct dualpi2_sched_data *q, struct tcf_proto *fl; int result; + cb->classified = DUALPI2_C_CLASSIC; + dualpi2_read_ect(skb); if (cb->ect & q->ecn_mask) { cb->classified = DUALPI2_C_L4S; @@ -359,10 +361,8 @@ static int dualpi2_skb_classify(struct dualpi2_sched_data *q, } fl = rcu_dereference_bh(q->tcf_filters); - if (!fl) { - cb->classified = DUALPI2_C_CLASSIC; + if (!fl) return NET_XMIT_SUCCESS; - } result = tcf_classify(skb, NULL, fl, &res, false); if (result >= 0) { @@ -403,28 +403,28 @@ static int dualpi2_enqueue_skb(struct sk_buff *skb, struct Qdisc *sch, cb = dualpi2_skb_cb(skb); cb->ts = ktime_get_ns(); - q->memory_used += skb->truesize; + WRITE_ONCE(q->memory_used, q->memory_used + skb->truesize); if (q->memory_used > q->max_memory_used) - q->max_memory_used = q->memory_used; + WRITE_ONCE(q->max_memory_used, q->memory_used); if (qdisc_qlen(sch) > q->maxq) - q->maxq = qdisc_qlen(sch); + WRITE_ONCE(q->maxq, qdisc_qlen(sch)); if (skb_in_l_queue(skb)) { /* Apply step thresh if skb is L4S && L-queue len >= min_qlen */ dualpi2_skb_cb(skb)->apply_step = skb_apply_step(skb, q); /* Keep the overall qdisc stats consistent */ - ++sch->q.qlen; + qdisc_qlen_inc(sch); qdisc_qstats_backlog_inc(sch, skb); - ++q->packets_in_l; + WRITE_ONCE(q->packets_in_l, q->packets_in_l + 1); if (!q->l_head_ts) - q->l_head_ts = cb->ts; + WRITE_ONCE(q->l_head_ts, cb->ts); return qdisc_enqueue_tail(skb, q->l_queue); } - ++q->packets_in_c; + WRITE_ONCE(q->packets_in_c, q->packets_in_c + 1); if (!q->c_head_ts) - q->c_head_ts = cb->ts; + WRITE_ONCE(q->c_head_ts, cb->ts); return qdisc_enqueue_tail(skb, sch); } @@ -529,18 +529,17 @@ static struct sk_buff *dequeue_packet(struct Qdisc *sch, qdisc_qstats_backlog_dec(q->l_queue, skb); /* Keep the global queue size consistent */ - --sch->q.qlen; - q->memory_used -= skb->truesize; + qdisc_qlen_dec(sch); } else if (c_len) { skb = __qdisc_dequeue_head(&sch->q); WRITE_ONCE(q->c_head_ts, head_enqueue_time(sch)); if (qdisc_qlen(q->l_queue)) *credit_change = ~((s32)q->c_protection_wl) + 1; - q->memory_used -= skb->truesize; } else { dualpi2_reset_c_protection(q); return NULL; } + WRITE_ONCE(q->memory_used, q->memory_used - skb->truesize); *credit_change *= qdisc_pkt_len(skb); qdisc_qstats_backlog_dec(sch, skb); return skb; @@ -563,7 +562,7 @@ static int do_step_aqm(struct dualpi2_sched_data *q, struct sk_buff *skb, } if (dualpi2_mark(q, skb)) - ++q->step_marks; + WRITE_ONCE(q->step_marks, q->step_marks + 1); } qdisc_bstats_update(q->l_queue, skb); return 0; @@ -578,7 +577,7 @@ static void drop_and_retry(struct dualpi2_sched_data *q, struct sk_buff *skb, qdisc_qstats_drop(sch); } -static struct sk_buff *dualpi2_qdisc_dequeue(struct Qdisc *sch) +static struct sk_buff *__dualpi2_qdisc_dequeue(struct Qdisc *sch) { struct dualpi2_sched_data *q = qdisc_priv(sch); struct sk_buff *skb; @@ -599,17 +598,55 @@ static struct sk_buff *dualpi2_qdisc_dequeue(struct Qdisc *sch) continue; } - q->c_protection_credit += credit_change; + WRITE_ONCE(q->c_protection_credit, + q->c_protection_credit + credit_change); qdisc_bstats_update(sch, skb); break; } + return skb; +} + +static void dualpi2_dequeue_drop(struct Qdisc *sch) +{ + struct dualpi2_sched_data *q = qdisc_priv(sch); + if (q->deferred_drops_cnt) { qdisc_tree_reduce_backlog(sch, q->deferred_drops_cnt, q->deferred_drops_len); q->deferred_drops_cnt = 0; q->deferred_drops_len = 0; } +} + +static struct sk_buff *dualpi2_qdisc_dequeue(struct Qdisc *sch) +{ + struct sk_buff *skb; + + skb = __dualpi2_qdisc_dequeue(sch); + + dualpi2_dequeue_drop(sch); + + return skb; +} + +static struct sk_buff *dualpi2_peek(struct Qdisc *sch) +{ + struct sk_buff *skb = skb_peek(&sch->gso_skb); + + if (!skb) { + skb = __dualpi2_qdisc_dequeue(sch); + + if (skb) { + __skb_queue_head(&sch->gso_skb, skb); + /* it's still part of the queue */ + qdisc_qstats_backlog_inc(sch, skb); + sch->q.qlen++; + } + + dualpi2_dequeue_drop(sch); + } + return skb; } @@ -875,7 +912,7 @@ static int dualpi2_change(struct Qdisc *sch, struct nlattr *opt, WARN_ON_ONCE(1); break; } - q->memory_used -= skb->truesize; + WRITE_ONCE(q->memory_used, q->memory_used - skb->truesize); rtnl_qdisc_drop(skb, sch); } else if (qdisc_qlen(q->l_queue)) { skb = qdisc_dequeue_internal(q->l_queue, true); @@ -887,9 +924,9 @@ static int dualpi2_change(struct Qdisc *sch, struct nlattr *opt, * l_queue on enqueue; qdisc_dequeue_internal() * handled l_queue, so we further account for sch. */ - --sch->q.qlen; + qdisc_qlen_dec(sch); qdisc_qstats_backlog_dec(sch, skb); - q->memory_used -= skb->truesize; + WRITE_ONCE(q->memory_used, q->memory_used - skb->truesize); rtnl_qdisc_drop(skb, q->l_queue); qdisc_qstats_drop(sch); } else { @@ -1045,15 +1082,15 @@ static int dualpi2_dump_stats(struct Qdisc *sch, struct gnet_dump *d) struct dualpi2_sched_data *q = qdisc_priv(sch); struct tc_dualpi2_xstats st = { .prob = READ_ONCE(q->pi2_prob), - .packets_in_c = q->packets_in_c, - .packets_in_l = q->packets_in_l, - .maxq = q->maxq, - .ecn_mark = q->ecn_mark, - .credit = q->c_protection_credit, - .step_marks = q->step_marks, - .memory_used = q->memory_used, - .max_memory_used = q->max_memory_used, - .memory_limit = q->memory_limit, + .packets_in_c = READ_ONCE(q->packets_in_c), + .packets_in_l = READ_ONCE(q->packets_in_l), + .maxq = READ_ONCE(q->maxq), + .ecn_mark = READ_ONCE(q->ecn_mark), + .credit = READ_ONCE(q->c_protection_credit), + .step_marks = READ_ONCE(q->step_marks), + .memory_used = READ_ONCE(q->memory_used), + .max_memory_used = READ_ONCE(q->max_memory_used), + .memory_limit = READ_ONCE(q->memory_limit), }; u64 qc, ql; @@ -1073,16 +1110,16 @@ static void dualpi2_reset(struct Qdisc *sch) qdisc_reset_queue(sch); qdisc_reset_queue(q->l_queue); - q->c_head_ts = 0; - q->l_head_ts = 0; - q->pi2_prob = 0; - q->packets_in_c = 0; - q->packets_in_l = 0; - q->maxq = 0; - q->ecn_mark = 0; - q->step_marks = 0; - q->memory_used = 0; - q->max_memory_used = 0; + WRITE_ONCE(q->c_head_ts, 0); + WRITE_ONCE(q->l_head_ts, 0); + WRITE_ONCE(q->pi2_prob, 0); + WRITE_ONCE(q->packets_in_c, 0); + WRITE_ONCE(q->packets_in_l, 0); + WRITE_ONCE(q->maxq, 0); + WRITE_ONCE(q->ecn_mark, 0); + WRITE_ONCE(q->step_marks, 0); + WRITE_ONCE(q->memory_used, 0); + WRITE_ONCE(q->max_memory_used, 0); dualpi2_reset_c_protection(q); } @@ -1164,7 +1201,7 @@ static struct Qdisc_ops dualpi2_qdisc_ops __read_mostly = { .priv_size = sizeof(struct dualpi2_sched_data), .enqueue = dualpi2_qdisc_enqueue, .dequeue = dualpi2_qdisc_dequeue, - .peek = qdisc_peek_dequeued, + .peek = dualpi2_peek, .init = dualpi2_init, .destroy = dualpi2_destroy, .reset = dualpi2_reset, @@ -1173,6 +1210,7 @@ static struct Qdisc_ops dualpi2_qdisc_ops __read_mostly = { .dump_stats = dualpi2_dump_stats, .owner = THIS_MODULE, }; +MODULE_ALIAS_NET_SCH("dualpi2"); static int __init dualpi2_module_init(void) { diff --git a/net/sched/sch_etf.c b/net/sched/sch_etf.c index c74d778c32a1..ada87a81da6a 100644 --- a/net/sched/sch_etf.c +++ b/net/sched/sch_etf.c @@ -189,7 +189,7 @@ static int etf_enqueue_timesortedlist(struct sk_buff *nskb, struct Qdisc *sch, rb_insert_color_cached(&nskb->rbnode, &q->head, leftmost); qdisc_qstats_backlog_inc(sch, nskb); - sch->q.qlen++; + qdisc_qlen_inc(sch); /* Now we may need to re-arm the qdisc watchdog for the next packet. */ reset_watchdog(sch); @@ -222,7 +222,7 @@ static void timesortedlist_drop(struct Qdisc *sch, struct sk_buff *skb, qdisc_qstats_backlog_dec(sch, skb); qdisc_drop(skb, sch, &to_free); qdisc_qstats_overlimit(sch); - sch->q.qlen--; + qdisc_qlen_dec(sch); } kfree_skb_list(to_free); @@ -247,7 +247,7 @@ static void timesortedlist_remove(struct Qdisc *sch, struct sk_buff *skb) q->last = skb->tstamp; - sch->q.qlen--; + qdisc_qlen_dec(sch); } static struct sk_buff *etf_dequeue_timesortedlist(struct Qdisc *sch) @@ -426,7 +426,7 @@ static void timesortedlist_clear(struct Qdisc *sch) rb_erase_cached(&skb->rbnode, &q->head); rtnl_kfree_skbs(skb, skb); - sch->q.qlen--; + qdisc_qlen_dec(sch); } } diff --git a/net/sched/sch_ets.c b/net/sched/sch_ets.c index a4b07b661b77..c817e0a6c146 100644 --- a/net/sched/sch_ets.c +++ b/net/sched/sch_ets.c @@ -449,7 +449,7 @@ static int ets_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch, } sch->qstats.backlog += len; - sch->q.qlen++; + qdisc_qlen_inc(sch); return err; } @@ -458,7 +458,7 @@ ets_qdisc_dequeue_skb(struct Qdisc *sch, struct sk_buff *skb) { qdisc_bstats_update(sch, skb); qdisc_qstats_backlog_dec(sch, skb); - sch->q.qlen--; + qdisc_qlen_dec(sch); return skb; } diff --git a/net/sched/sch_fq.c b/net/sched/sch_fq.c index f2edcf872981..1e34ac136b15 100644 --- a/net/sched/sch_fq.c +++ b/net/sched/sch_fq.c @@ -497,7 +497,7 @@ static void fq_dequeue_skb(struct Qdisc *sch, struct fq_flow *flow, fq_erase_head(sch, flow, skb); skb_mark_not_on_list(skb); qdisc_qstats_backlog_dec(sch, skb); - sch->q.qlen--; + qdisc_qlen_dec(sch); qdisc_bstats_update(sch, skb); } @@ -597,7 +597,7 @@ static int fq_enqueue(struct sk_buff *skb, struct Qdisc *sch, flow_queue_add(f, skb); qdisc_qstats_backlog_inc(sch, skb); - sch->q.qlen++; + qdisc_qlen_inc(sch); return NET_XMIT_SUCCESS; } @@ -801,7 +801,7 @@ static void fq_reset(struct Qdisc *sch) struct fq_flow *f; unsigned int idx; - sch->q.qlen = 0; + WRITE_ONCE(sch->q.qlen, 0); sch->qstats.backlog = 0; fq_flow_purge(&q->internal); diff --git a/net/sched/sch_fq_codel.c b/net/sched/sch_fq_codel.c index 24db54684e8a..09ab84580160 100644 --- a/net/sched/sch_fq_codel.c +++ b/net/sched/sch_fq_codel.c @@ -178,7 +178,7 @@ static unsigned int fq_codel_drop(struct Qdisc *sch, unsigned int max_packets, q->memory_usage -= mem; sch->qstats.drops += i; sch->qstats.backlog -= len; - sch->q.qlen -= i; + WRITE_ONCE(sch->q.qlen, sch->q.qlen - i); return idx; } @@ -215,7 +215,8 @@ static int fq_codel_enqueue(struct sk_buff *skb, struct Qdisc *sch, get_codel_cb(skb)->mem_usage = skb->truesize; q->memory_usage += get_codel_cb(skb)->mem_usage; memory_limited = q->memory_usage > q->memory_limit; - if (++sch->q.qlen <= sch->limit && !memory_limited) + qdisc_qlen_inc(sch); + if (sch->q.qlen <= sch->limit && !memory_limited) return NET_XMIT_SUCCESS; prev_backlog = sch->qstats.backlog; @@ -266,7 +267,7 @@ static struct sk_buff *dequeue_func(struct codel_vars *vars, void *ctx) WRITE_ONCE(q->backlogs[flow - q->flows], q->backlogs[flow - q->flows] - qdisc_pkt_len(skb)); q->memory_usage -= get_codel_cb(skb)->mem_usage; - sch->q.qlen--; + qdisc_qlen_dec(sch); sch->qstats.backlog -= qdisc_pkt_len(skb); } return skb; @@ -280,7 +281,7 @@ static void drop_func(struct sk_buff *skb, void *ctx) qdisc_qstats_drop(sch); } -static struct sk_buff *fq_codel_dequeue(struct Qdisc *sch) +static struct sk_buff *__fq_codel_dequeue(struct Qdisc *sch) { struct fq_codel_sched_data *q = qdisc_priv(sch); struct sk_buff *skb; @@ -317,12 +318,49 @@ begin: qdisc_bstats_update(sch, skb); WRITE_ONCE(flow->deficit, flow->deficit - qdisc_pkt_len(skb)); + return skb; +} + +static void fq_codel_dequeue_drop(struct Qdisc *sch) +{ + struct fq_codel_sched_data *q = qdisc_priv(sch); + if (q->cstats.drop_count) { qdisc_tree_reduce_backlog(sch, q->cstats.drop_count, q->cstats.drop_len); q->cstats.drop_count = 0; q->cstats.drop_len = 0; } +} + +static struct sk_buff *fq_codel_dequeue(struct Qdisc *sch) +{ + struct sk_buff *skb; + + skb = __fq_codel_dequeue(sch); + + fq_codel_dequeue_drop(sch); + + return skb; +} + +static struct sk_buff *fq_codel_peek(struct Qdisc *sch) +{ + struct sk_buff *skb = skb_peek(&sch->gso_skb); + + if (!skb) { + skb = __fq_codel_dequeue(sch); + + if (skb) { + __skb_queue_head(&sch->gso_skb, skb); + /* it's still part of the queue */ + qdisc_qstats_backlog_inc(sch, skb); + sch->q.qlen++; + } + + fq_codel_dequeue_drop(sch); + } + return skb; } @@ -725,7 +763,7 @@ static struct Qdisc_ops fq_codel_qdisc_ops __read_mostly = { .priv_size = sizeof(struct fq_codel_sched_data), .enqueue = fq_codel_enqueue, .dequeue = fq_codel_dequeue, - .peek = qdisc_peek_dequeued, + .peek = fq_codel_peek, .init = fq_codel_init, .reset = fq_codel_reset, .destroy = fq_codel_destroy, diff --git a/net/sched/sch_fq_pie.c b/net/sched/sch_fq_pie.c index 7becbf5362b3..0a4eca4ab086 100644 --- a/net/sched/sch_fq_pie.c +++ b/net/sched/sch_fq_pie.c @@ -185,7 +185,7 @@ static int fq_pie_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch, q->stats.packets_in++; q->memory_usage += skb->truesize; sch->qstats.backlog += pkt_len; - sch->q.qlen++; + qdisc_qlen_inc(sch); flow_queue_add(sel_flow, skb); if (list_empty(&sel_flow->flowchain)) { list_add_tail(&sel_flow->flowchain, &q->new_flows); @@ -263,7 +263,7 @@ begin: skb = dequeue_head(flow); pkt_len = qdisc_pkt_len(skb); sch->qstats.backlog -= pkt_len; - sch->q.qlen--; + qdisc_qlen_dec(sch); qdisc_bstats_update(sch, skb); } diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c index a93321db8fd7..fe73ac40d409 100644 --- a/net/sched/sch_generic.c +++ b/net/sched/sch_generic.c @@ -118,7 +118,7 @@ static inline struct sk_buff *__skb_dequeue_bad_txq(struct Qdisc *q) qdisc_qstats_cpu_qlen_dec(q); } else { qdisc_qstats_backlog_dec(q, skb); - q->q.qlen--; + qdisc_qlen_dec(q); } } else { skb = SKB_XOFF_MAGIC; @@ -159,7 +159,7 @@ static inline void qdisc_enqueue_skb_bad_txq(struct Qdisc *q, qdisc_qstats_cpu_qlen_inc(q); } else { qdisc_qstats_backlog_inc(q, skb); - q->q.qlen++; + qdisc_qlen_inc(q); } if (lock) @@ -188,7 +188,7 @@ static inline void dev_requeue_skb(struct sk_buff *skb, struct Qdisc *q) } else { q->qstats.requeues++; qdisc_qstats_backlog_inc(q, skb); - q->q.qlen++; + qdisc_qlen_inc(q); } skb = next; @@ -294,7 +294,7 @@ static struct sk_buff *dequeue_skb(struct Qdisc *q, bool *validate, qdisc_qstats_cpu_qlen_dec(q); } else { qdisc_qstats_backlog_dec(q, skb); - q->q.qlen--; + qdisc_qlen_dec(q); } } else { skb = NULL; @@ -568,16 +568,24 @@ static void dev_watchdog(struct timer_list *t) dev->netdev_ops->ndo_tx_timeout(dev, i); netif_unfreeze_queues(dev); } - if (!mod_timer(&dev->watchdog_timer, - round_jiffies(oldest_start + - dev->watchdog_timeo))) - release = false; + spin_lock(&dev->watchdog_lock); + mod_timer(&dev->watchdog_timer, + round_jiffies(oldest_start + + dev->watchdog_timeo)); + release = false; + spin_unlock(&dev->watchdog_lock); } } spin_unlock(&dev->tx_global_lock); - if (release) + spin_lock(&dev->watchdog_lock); + if (timer_pending(&dev->watchdog_timer)) + release = false; + if (release && dev->watchdog_ref_held) { netdev_put(dev, &dev->watchdog_dev_tracker); + dev->watchdog_ref_held = false; + } + spin_unlock(&dev->watchdog_lock); } void netdev_watchdog_up(struct net_device *dev) @@ -586,18 +594,31 @@ void netdev_watchdog_up(struct net_device *dev) return; if (dev->watchdog_timeo <= 0) dev->watchdog_timeo = 5*HZ; + + spin_lock_bh(&dev->watchdog_lock); if (!mod_timer(&dev->watchdog_timer, - round_jiffies(jiffies + dev->watchdog_timeo))) - netdev_hold(dev, &dev->watchdog_dev_tracker, - GFP_ATOMIC); + round_jiffies(jiffies + dev->watchdog_timeo))) { + if (!dev->watchdog_ref_held) { + netdev_hold(dev, &dev->watchdog_dev_tracker, + GFP_ATOMIC); + dev->watchdog_ref_held = true; + } + } + spin_unlock_bh(&dev->watchdog_lock); } EXPORT_SYMBOL_GPL(netdev_watchdog_up); static void netdev_watchdog_down(struct net_device *dev) { netif_tx_lock_bh(dev); - if (timer_delete(&dev->watchdog_timer)) + + spin_lock(&dev->watchdog_lock); + if (timer_delete(&dev->watchdog_timer)) { netdev_put(dev, &dev->watchdog_dev_tracker); + dev->watchdog_ref_held = false; + } + spin_unlock(&dev->watchdog_lock); + netif_tx_unlock_bh(dev); } @@ -614,8 +635,6 @@ void netif_carrier_on(struct net_device *dev) return; atomic_inc(&dev->carrier_up_count); linkwatch_fire_event(dev); - if (netif_running(dev)) - netdev_watchdog_up(dev); } } EXPORT_SYMBOL(netif_carrier_on); @@ -1059,7 +1078,7 @@ void qdisc_reset(struct Qdisc *qdisc) __skb_queue_purge(&qdisc->gso_skb); __skb_queue_purge(&qdisc->skb_bad_txq); - qdisc->q.qlen = 0; + WRITE_ONCE(qdisc->q.qlen, 0); qdisc->qstats.backlog = 0; } EXPORT_SYMBOL(qdisc_reset); diff --git a/net/sched/sch_hfsc.c b/net/sched/sch_hfsc.c index 83b2ca2e37fc..50112c8a71e0 100644 --- a/net/sched/sch_hfsc.c +++ b/net/sched/sch_hfsc.c @@ -715,7 +715,7 @@ init_vf(struct hfsc_class *cl, unsigned int len) rtsc_min(&cl->cl_virtual, &cl->cl_fsc, cl->cl_vt, cl->cl_total); cl->cl_vtadj = 0; - cl->cl_vtperiod++; /* increment vt period */ + WRITE_ONCE(cl->cl_vtperiod, cl->cl_vtperiod + 1); /* increment vt period */ cl->cl_parentperiod = cl->cl_parent->cl_vtperiod; if (cl->cl_parent->cl_nactive == 0) cl->cl_parentperiod++; @@ -753,11 +753,11 @@ update_vf(struct hfsc_class *cl, unsigned int len, u64 cur_time) u64 f; /* , myf_bound, delta; */ int go_passive = 0; - if (cl->qdisc->q.qlen == 0 && cl->cl_flags & HFSC_FSC) + if (cl->qdisc->q.qlen == 0 && cl->cl_flags & HFSC_FSC && cl->cl_nactive) go_passive = 1; for (; cl->cl_parent != NULL; cl = cl->cl_parent) { - cl->cl_total += len; + WRITE_ONCE(cl->cl_total, cl->cl_total + len); if (!(cl->cl_flags & HFSC_FSC) || cl->cl_nactive == 0) continue; @@ -847,7 +847,7 @@ hfsc_adjust_levels(struct hfsc_class *cl) if (p->level >= level) level = p->level + 1; } - cl->level = level; + WRITE_ONCE(cl->level, level); } while ((cl = cl->cl_parent) != NULL); } @@ -1338,10 +1338,10 @@ hfsc_dump_class_stats(struct Qdisc *sch, unsigned long arg, __u32 qlen; qdisc_qstats_qlen_backlog(cl->qdisc, &qlen, &cl->qstats.backlog); - xstats.level = cl->level; - xstats.period = cl->cl_vtperiod; - xstats.work = cl->cl_total; - xstats.rtwork = cl->cl_cumul; + xstats.level = READ_ONCE(cl->level); + xstats.period = READ_ONCE(cl->cl_vtperiod); + xstats.work = READ_ONCE(cl->cl_total); + xstats.rtwork = READ_ONCE(cl->cl_cumul); if (gnet_stats_copy_basic(d, NULL, &cl->bstats, true) < 0 || gnet_stats_copy_rate_est(d, &cl->rate_est) < 0 || @@ -1452,15 +1452,15 @@ hfsc_change_qdisc(struct Qdisc *sch, struct nlattr *opt, static void hfsc_reset_class(struct hfsc_class *cl) { - cl->cl_total = 0; - cl->cl_cumul = 0; + WRITE_ONCE(cl->cl_total, 0); + WRITE_ONCE(cl->cl_cumul, 0); cl->cl_d = 0; cl->cl_e = 0; cl->cl_vt = 0; cl->cl_vtadj = 0; cl->cl_cvtmin = 0; cl->cl_cvtoff = 0; - cl->cl_vtperiod = 0; + WRITE_ONCE(cl->cl_vtperiod, 0); cl->cl_parentperiod = 0; cl->cl_f = 0; cl->cl_myf = 0; @@ -1561,7 +1561,7 @@ hfsc_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free) } sch->qstats.backlog += len; - sch->q.qlen++; + qdisc_qlen_inc(sch); if (first && !cl_in_el_or_vttree(cl)) { if (cl->cl_flags & HFSC_RSC) @@ -1626,7 +1626,7 @@ hfsc_dequeue(struct Qdisc *sch) bstats_update(&cl->bstats, skb); update_vf(cl, qdisc_pkt_len(skb), cur_time); if (realtime) - cl->cl_cumul += qdisc_pkt_len(skb); + WRITE_ONCE(cl->cl_cumul, cl->cl_cumul + qdisc_pkt_len(skb)); if (cl->cl_flags & HFSC_RSC) { if (cl->qdisc->q.qlen != 0) { @@ -1650,7 +1650,7 @@ hfsc_dequeue(struct Qdisc *sch) qdisc_bstats_update(sch, skb); qdisc_qstats_backlog_dec(sch, skb); - sch->q.qlen--; + qdisc_qlen_dec(sch); return skb; } diff --git a/net/sched/sch_hhf.c b/net/sched/sch_hhf.c index 96021f52d835..d85cb0263b67 100644 --- a/net/sched/sch_hhf.c +++ b/net/sched/sch_hhf.c @@ -360,7 +360,7 @@ static unsigned int hhf_drop(struct Qdisc *sch, struct sk_buff **to_free) if (bucket->head) { struct sk_buff *skb = dequeue_head(bucket); - sch->q.qlen--; + qdisc_qlen_dec(sch); qdisc_qstats_backlog_dec(sch, skb); qdisc_drop(skb, sch, to_free); } @@ -400,7 +400,8 @@ static int hhf_enqueue(struct sk_buff *skb, struct Qdisc *sch, } bucket->deficit = weight * q->quantum; } - if (++sch->q.qlen <= sch->limit) + qdisc_qlen_inc(sch); + if (sch->q.qlen <= sch->limit) return NET_XMIT_SUCCESS; prev_backlog = sch->qstats.backlog; @@ -443,7 +444,7 @@ begin: if (bucket->head) { skb = dequeue_head(bucket); - sch->q.qlen--; + qdisc_qlen_dec(sch); qdisc_qstats_backlog_dec(sch, skb); } @@ -461,12 +462,39 @@ begin: return skb; } +static void hhf_reset_classifier(struct hhf_sched_data *q) +{ + int i; + + if (!q->hh_flows) + return; + + for (i = 0; i < HH_FLOWS_CNT; i++) { + struct hh_flow_state *flow, *next; + struct list_head *head = &q->hh_flows[i]; + + list_for_each_entry_safe(flow, next, head, flowchain) { + list_del(&flow->flowchain); + kfree(flow); + } + } + WRITE_ONCE(q->hh_flows_current_cnt, 0); + + for (i = 0; i < HHF_ARRAYS_CNT; i++) { + if (q->hhf_valid_bits[i]) + bitmap_zero(q->hhf_valid_bits[i], HHF_ARRAYS_LEN); + } + q->hhf_arrays_reset_timestamp = hhf_time_stamp(); +} + static void hhf_reset(struct Qdisc *sch) { + struct hhf_sched_data *q = qdisc_priv(sch); struct sk_buff *skb; while ((skb = hhf_dequeue(sch)) != NULL) rtnl_kfree_skbs(skb, skb); + hhf_reset_classifier(q); } static void hhf_destroy(struct Qdisc *sch) diff --git a/net/sched/sch_htb.c b/net/sched/sch_htb.c index eb12381795ce..ccabafed3fe4 100644 --- a/net/sched/sch_htb.c +++ b/net/sched/sch_htb.c @@ -568,7 +568,7 @@ htb_change_class_mode(struct htb_sched *q, struct htb_class *cl, s64 *diff) if (new_mode == HTB_CANT_SEND) { cl->overlimits++; - q->overlimits++; + WRITE_ONCE(q->overlimits, q->overlimits + 1); } if (cl->prio_activity) { /* not necessary: speed optimization */ @@ -628,7 +628,7 @@ static int htb_enqueue(struct sk_buff *skb, struct Qdisc *sch, /* enqueue to helper queue */ if (q->direct_queue.qlen < q->direct_qlen) { __qdisc_enqueue_tail(skb, &q->direct_queue); - q->direct_pkts++; + WRITE_ONCE(q->direct_pkts, q->direct_pkts + 1); } else { return qdisc_drop(skb, sch, to_free); } @@ -651,7 +651,7 @@ static int htb_enqueue(struct sk_buff *skb, struct Qdisc *sch, } sch->qstats.backlog += len; - sch->q.qlen++; + qdisc_qlen_inc(sch); return NET_XMIT_SUCCESS; } @@ -951,7 +951,7 @@ static struct sk_buff *htb_dequeue(struct Qdisc *sch) ok: qdisc_bstats_update(sch, skb); qdisc_qstats_backlog_dec(sch, skb); - sch->q.qlen--; + qdisc_qlen_dec(sch); return skb; } @@ -1147,6 +1147,7 @@ static int htb_init(struct Qdisc *sch, struct nlattr *opt, * parts (especially calling ndo_setup_tc) on errors. */ q->offload = true; + sch->flags |= TCQ_F_OFFLOADED; return 0; } @@ -1207,17 +1208,12 @@ static int htb_dump(struct Qdisc *sch, struct sk_buff *skb) struct nlattr *nest; struct tc_htb_glob gopt; - if (q->offload) - sch->flags |= TCQ_F_OFFLOADED; - else - sch->flags &= ~TCQ_F_OFFLOADED; - - sch->qstats.overlimits = q->overlimits; + sch->qstats.overlimits = READ_ONCE(q->overlimits); /* Its safe to not acquire qdisc lock. As we hold RTNL, * no change can happen on the qdisc parameters. */ - gopt.direct_pkts = q->direct_pkts; + gopt.direct_pkts = READ_ONCE(q->direct_pkts); gopt.version = HTB_VER; gopt.rate2quantum = q->rate2quantum; gopt.defcls = q->defcls; diff --git a/net/sched/sch_mq.c b/net/sched/sch_mq.c index a0133a7b9d3b..ec8c91d3fde0 100644 --- a/net/sched/sch_mq.c +++ b/net/sched/sch_mq.c @@ -143,10 +143,10 @@ EXPORT_SYMBOL_NS_GPL(mq_attach, "NET_SCHED_INTERNAL"); void mq_dump_common(struct Qdisc *sch, struct sk_buff *skb) { struct net_device *dev = qdisc_dev(sch); + unsigned int qlen = 0; struct Qdisc *qdisc; unsigned int ntx; - sch->q.qlen = 0; gnet_stats_basic_sync_init(&sch->bstats); memset(&sch->qstats, 0, sizeof(sch->qstats)); @@ -163,10 +163,11 @@ void mq_dump_common(struct Qdisc *sch, struct sk_buff *skb) &qdisc->bstats, false); gnet_stats_add_queue(&sch->qstats, qdisc->cpu_qstats, &qdisc->qstats); - sch->q.qlen += qdisc_qlen(qdisc); + qlen += qdisc_qlen(qdisc); spin_unlock_bh(qdisc_lock(qdisc)); } + WRITE_ONCE(sch->q.qlen, qlen); } EXPORT_SYMBOL_NS_GPL(mq_dump_common, "NET_SCHED_INTERNAL"); diff --git a/net/sched/sch_mqprio.c b/net/sched/sch_mqprio.c index 002add5ce9e0..91a92992cd24 100644 --- a/net/sched/sch_mqprio.c +++ b/net/sched/sch_mqprio.c @@ -555,10 +555,11 @@ static int mqprio_dump(struct Qdisc *sch, struct sk_buff *skb) struct mqprio_sched *priv = qdisc_priv(sch); struct nlattr *nla = (struct nlattr *)skb_tail_pointer(skb); struct tc_mqprio_qopt opt = { 0 }; + unsigned int qlen = 0; struct Qdisc *qdisc; unsigned int ntx; - sch->q.qlen = 0; + qlen = 0; gnet_stats_basic_sync_init(&sch->bstats); memset(&sch->qstats, 0, sizeof(sch->qstats)); @@ -575,10 +576,11 @@ static int mqprio_dump(struct Qdisc *sch, struct sk_buff *skb) &qdisc->bstats, false); gnet_stats_add_queue(&sch->qstats, qdisc->cpu_qstats, &qdisc->qstats); - sch->q.qlen += qdisc_qlen(qdisc); + qlen += qdisc_qlen(qdisc); spin_unlock_bh(qdisc_lock(qdisc)); } + WRITE_ONCE(sch->q.qlen, qlen); mqprio_qopt_reconstruct(dev, &opt); opt.hw = priv->hw_offload; @@ -663,12 +665,12 @@ static int mqprio_dump_class_stats(struct Qdisc *sch, unsigned long cl, __acquires(d->lock) { if (cl >= TC_H_MIN_PRIORITY) { - int i; - __u32 qlen; - struct gnet_stats_queue qstats = {0}; - struct gnet_stats_basic_sync bstats; struct net_device *dev = qdisc_dev(sch); struct netdev_tc_txq tc = dev->tc_to_txq[cl & TC_BITMASK]; + struct gnet_stats_queue qstats = {0}; + struct gnet_stats_basic_sync bstats; + u32 qlen = 0; + int i; gnet_stats_basic_sync_init(&bstats); /* Drop lock here it will be reclaimed before touching @@ -689,11 +691,11 @@ static int mqprio_dump_class_stats(struct Qdisc *sch, unsigned long cl, &qdisc->bstats, false); gnet_stats_add_queue(&qstats, qdisc->cpu_qstats, &qdisc->qstats); - sch->q.qlen += qdisc_qlen(qdisc); + qlen += qdisc_qlen(qdisc); spin_unlock_bh(qdisc_lock(qdisc)); } - qlen = qdisc_qlen(sch) + qstats.qlen; + qlen = qlen + qstats.qlen; /* Reclaim root sleeping lock before completing stats */ if (d->lock) diff --git a/net/sched/sch_multiq.c b/net/sched/sch_multiq.c index 9f822fee113d..a467dd122369 100644 --- a/net/sched/sch_multiq.c +++ b/net/sched/sch_multiq.c @@ -76,7 +76,7 @@ multiq_enqueue(struct sk_buff *skb, struct Qdisc *sch, ret = qdisc_enqueue(skb, qdisc, to_free); if (ret == NET_XMIT_SUCCESS) { - sch->q.qlen++; + qdisc_qlen_inc(sch); return NET_XMIT_SUCCESS; } if (net_xmit_drop_count(ret)) @@ -103,10 +103,10 @@ static struct sk_buff *multiq_dequeue(struct Qdisc *sch) if (!netif_xmit_stopped( netdev_get_tx_queue(qdisc_dev(sch), q->curband))) { qdisc = q->queues[q->curband]; - skb = qdisc->dequeue(qdisc); + skb = qdisc_dequeue_peeked(qdisc); if (skb) { qdisc_bstats_update(sch, skb); - sch->q.qlen--; + qdisc_qlen_dec(sch); return skb; } } diff --git a/net/sched/sch_netem.c b/net/sched/sch_netem.c index 17a79fe2f091..db44cdb47dce 100644 --- a/net/sched/sch_netem.c +++ b/net/sched/sch_netem.c @@ -416,7 +416,7 @@ static void tfifo_enqueue(struct sk_buff *nskb, struct Qdisc *sch) rb_insert_color(&nskb->rbnode, &q->t_root); } q->t_len++; - sch->q.qlen++; + qdisc_qlen_inc(sch); } /* netem can't properly corrupt a megapacket (like we get from GSO), so instead @@ -750,19 +750,19 @@ deliver: if (net_xmit_drop_count(err)) qdisc_qstats_drop(sch); sch->qstats.backlog -= pkt_len; - sch->q.qlen--; + qdisc_qlen_dec(sch); qdisc_tree_reduce_backlog(sch, 1, pkt_len); } goto tfifo_dequeue; } - sch->q.qlen--; + qdisc_qlen_dec(sch); goto deliver; } if (q->qdisc) { skb = q->qdisc->ops->dequeue(q->qdisc); if (skb) { - sch->q.qlen--; + qdisc_qlen_dec(sch); goto deliver; } } @@ -775,7 +775,7 @@ deliver: if (q->qdisc) { skb = q->qdisc->ops->dequeue(q->qdisc); if (skb) { - sch->q.qlen--; + qdisc_qlen_dec(sch); goto deliver; } } diff --git a/net/sched/sch_prio.c b/net/sched/sch_prio.c index 9e2b9a490db2..fe42ae3d6b69 100644 --- a/net/sched/sch_prio.c +++ b/net/sched/sch_prio.c @@ -86,7 +86,7 @@ prio_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free) ret = qdisc_enqueue(skb, qdisc, to_free); if (ret == NET_XMIT_SUCCESS) { sch->qstats.backlog += len; - sch->q.qlen++; + qdisc_qlen_inc(sch); return NET_XMIT_SUCCESS; } if (net_xmit_drop_count(ret)) @@ -119,7 +119,7 @@ static struct sk_buff *prio_dequeue(struct Qdisc *sch) if (skb) { qdisc_bstats_update(sch, skb); qdisc_qstats_backlog_dec(sch, skb); - sch->q.qlen--; + qdisc_qlen_dec(sch); return skb; } } diff --git a/net/sched/sch_qfq.c b/net/sched/sch_qfq.c index 699e45873f86..195c434aae5f 100644 --- a/net/sched/sch_qfq.c +++ b/net/sched/sch_qfq.c @@ -1152,12 +1152,12 @@ static struct sk_buff *qfq_dequeue(struct Qdisc *sch) if (!skb) return NULL; - sch->q.qlen--; + qdisc_qlen_dec(sch); skb = agg_dequeue(in_serv_agg, cl, len); if (!skb) { - sch->q.qlen++; + qdisc_qlen_inc(sch); return NULL; } @@ -1265,7 +1265,7 @@ static int qfq_enqueue(struct sk_buff *skb, struct Qdisc *sch, _bstats_update(&cl->bstats, len, gso_segs); sch->qstats.backlog += len; - ++sch->q.qlen; + qdisc_qlen_inc(sch); agg = cl->agg; /* if the class is active, then done here */ diff --git a/net/sched/sch_red.c b/net/sched/sch_red.c index 4d0e44a2e7c6..0719590dfd73 100644 --- a/net/sched/sch_red.c +++ b/net/sched/sch_red.c @@ -139,7 +139,7 @@ static int red_enqueue(struct sk_buff *skb, struct Qdisc *sch, ret = qdisc_enqueue(skb, child, to_free); if (likely(ret == NET_XMIT_SUCCESS)) { sch->qstats.backlog += len; - sch->q.qlen++; + qdisc_qlen_inc(sch); } else if (net_xmit_drop_count(ret)) { WRITE_ONCE(q->stats.pdrop, q->stats.pdrop + 1); @@ -166,7 +166,7 @@ static struct sk_buff *red_dequeue(struct Qdisc *sch) if (skb) { qdisc_bstats_update(sch, skb); qdisc_qstats_backlog_dec(sch, skb); - sch->q.qlen--; + qdisc_qlen_dec(sch); } else { if (!red_is_idling(&q->vars)) red_start_of_idle_period(&q->vars); diff --git a/net/sched/sch_sfb.c b/net/sched/sch_sfb.c index d3ee8e5479b3..efd9251c3add 100644 --- a/net/sched/sch_sfb.c +++ b/net/sched/sch_sfb.c @@ -416,7 +416,7 @@ enqueue: ret = qdisc_enqueue(skb, child, to_free); if (likely(ret == NET_XMIT_SUCCESS)) { sch->qstats.backlog += len; - sch->q.qlen++; + qdisc_qlen_inc(sch); increment_qlen(&cb, q); } else if (net_xmit_drop_count(ret)) { WRITE_ONCE(q->stats.childdrop, @@ -446,7 +446,7 @@ static struct sk_buff *sfb_dequeue(struct Qdisc *sch) if (skb) { qdisc_bstats_update(sch, skb); qdisc_qstats_backlog_dec(sch, skb); - sch->q.qlen--; + qdisc_qlen_dec(sch); decrement_qlen(skb, q); } diff --git a/net/sched/sch_sfq.c b/net/sched/sch_sfq.c index f39822babf88..f9807ee2cf6c 100644 --- a/net/sched/sch_sfq.c +++ b/net/sched/sch_sfq.c @@ -302,7 +302,7 @@ drop: len = qdisc_pkt_len(skb); WRITE_ONCE(slot->backlog, slot->backlog - len); sfq_dec(q, x); - sch->q.qlen--; + qdisc_qlen_dec(sch); qdisc_qstats_backlog_dec(sch, skb); qdisc_drop_reason(skb, sch, to_free, QDISC_DROP_OVERLIMIT); return len; @@ -456,7 +456,8 @@ enqueue: /* We could use a bigger initial quantum for new flows */ WRITE_ONCE(slot->allot, q->quantum); } - if (++sch->q.qlen <= q->limit) + qdisc_qlen_inc(sch); + if (sch->q.qlen <= q->limit) return NET_XMIT_SUCCESS; qlen = slot->qlen; @@ -497,7 +498,7 @@ next_slot: skb = slot_dequeue_head(slot); sfq_dec(q, a); qdisc_bstats_update(sch, skb); - sch->q.qlen--; + qdisc_qlen_dec(sch); qdisc_qstats_backlog_dec(sch, skb); WRITE_ONCE(slot->backlog, slot->backlog - qdisc_pkt_len(skb)); /* Is the slot empty? */ @@ -596,7 +597,7 @@ drop: WRITE_ONCE(slot->allot, q->quantum); } } - sch->q.qlen -= dropped; + WRITE_ONCE(sch->q.qlen, sch->q.qlen - dropped); qdisc_tree_reduce_backlog(sch, dropped, drop_len); } diff --git a/net/sched/sch_skbprio.c b/net/sched/sch_skbprio.c index f485f62ab721..52abfb4015a3 100644 --- a/net/sched/sch_skbprio.c +++ b/net/sched/sch_skbprio.c @@ -93,7 +93,7 @@ static int skbprio_enqueue(struct sk_buff *skb, struct Qdisc *sch, if (prio < q->lowest_prio) q->lowest_prio = prio; - sch->q.qlen++; + qdisc_qlen_inc(sch); return NET_XMIT_SUCCESS; } @@ -145,7 +145,7 @@ static struct sk_buff *skbprio_dequeue(struct Qdisc *sch) if (unlikely(!skb)) return NULL; - sch->q.qlen--; + qdisc_qlen_dec(sch); qdisc_qstats_backlog_dec(sch, skb); qdisc_bstats_update(sch, skb); diff --git a/net/sched/sch_taprio.c b/net/sched/sch_taprio.c index 45245157e00a..d2d11a127f03 100644 --- a/net/sched/sch_taprio.c +++ b/net/sched/sch_taprio.c @@ -574,7 +574,7 @@ static int taprio_enqueue_one(struct sk_buff *skb, struct Qdisc *sch, } qdisc_qstats_backlog_inc(sch, skb); - sch->q.qlen++; + qdisc_qlen_inc(sch); return qdisc_enqueue(skb, child, to_free); } @@ -749,13 +749,13 @@ static struct sk_buff *taprio_dequeue_from_txq(struct Qdisc *sch, int txq, return NULL; skip_peek_checks: - skb = child->ops->dequeue(child); + skb = qdisc_dequeue_peeked(child); if (unlikely(!skb)) return NULL; qdisc_bstats_update(sch, skb); qdisc_qstats_backlog_dec(sch, skb); - sch->q.qlen--; + qdisc_qlen_dec(sch); return skb; } diff --git a/net/sched/sch_tbf.c b/net/sched/sch_tbf.c index f2340164f579..25edf11a7d67 100644 --- a/net/sched/sch_tbf.c +++ b/net/sched/sch_tbf.c @@ -231,7 +231,7 @@ static int tbf_segment(struct sk_buff *skb, struct Qdisc *sch, len += seg_len; } } - sch->q.qlen += nb; + WRITE_ONCE(sch->q.qlen, sch->q.qlen + nb); sch->qstats.backlog += len; if (nb > 0) { qdisc_tree_reduce_backlog(sch, 1 - nb, prev_len - len); @@ -264,7 +264,7 @@ static int tbf_enqueue(struct sk_buff *skb, struct Qdisc *sch, } sch->qstats.backlog += len; - sch->q.qlen++; + qdisc_qlen_inc(sch); return NET_XMIT_SUCCESS; } @@ -309,7 +309,7 @@ static struct sk_buff *tbf_dequeue(struct Qdisc *sch) q->tokens = toks; q->ptokens = ptoks; qdisc_qstats_backlog_dec(sch, skb); - sch->q.qlen--; + qdisc_qlen_dec(sch); qdisc_bstats_update(sch, skb); return skb; } diff --git a/net/sched/sch_teql.c b/net/sched/sch_teql.c index ec4039a201a2..5c42a29a981c 100644 --- a/net/sched/sch_teql.c +++ b/net/sched/sch_teql.c @@ -52,7 +52,8 @@ struct teql_master { struct Qdisc_ops qops; struct net_device *dev; - struct Qdisc *slaves; + struct Qdisc __rcu *slaves; + spinlock_t slaves_lock; /* serializes writes to ->slaves */ struct list_head master_list; unsigned long tx_bytes; unsigned long tx_packets; @@ -61,7 +62,7 @@ struct teql_master { }; struct teql_sched_data { - struct Qdisc *next; + struct Qdisc __rcu *next; struct teql_master *m; struct sk_buff_head q; }; @@ -101,13 +102,15 @@ teql_dequeue(struct Qdisc *sch) if (skb == NULL) { struct net_device *m = qdisc_dev(q); if (m) { - dat->m->slaves = sch; + spin_lock_bh(&dat->m->slaves_lock); + rcu_assign_pointer(dat->m->slaves, sch); + spin_unlock_bh(&dat->m->slaves_lock); netif_wake_queue(m); } } else { qdisc_bstats_update(sch, skb); } - sch->q.qlen = dat->q.qlen + q->q.qlen; + WRITE_ONCE(sch->q.qlen, dat->q.qlen + READ_ONCE(q->q.qlen)); return skb; } @@ -132,34 +135,49 @@ teql_destroy(struct Qdisc *sch) struct Qdisc *q, *prev; struct teql_sched_data *dat = qdisc_priv(sch); struct teql_master *master = dat->m; + struct netdev_queue *txq = NULL; + bool reset_master_queue = false; if (!master) return; - prev = master->slaves; + spin_lock_bh(&master->slaves_lock); + prev = rcu_dereference_protected(master->slaves, + lockdep_is_held(&master->slaves_lock)); if (prev) { do { - q = NEXT_SLAVE(prev); - if (q == sch) { - NEXT_SLAVE(prev) = NEXT_SLAVE(q); - if (q == master->slaves) { - master->slaves = NEXT_SLAVE(q); - if (q == master->slaves) { - struct netdev_queue *txq; - - txq = netdev_get_tx_queue(master->dev, 0); - master->slaves = NULL; - - dev_reset_queue(master->dev, - txq, NULL); - } - } - skb_queue_purge(&dat->q); - break; + struct Qdisc *head, *next; + + q = rcu_dereference_protected(NEXT_SLAVE(prev), + lockdep_is_held(&master->slaves_lock)); + if (q != sch) { + prev = q; + continue; } - } while ((prev = q) != master->slaves); + next = rcu_dereference_protected(NEXT_SLAVE(q), + lockdep_is_held(&master->slaves_lock)); + rcu_assign_pointer(NEXT_SLAVE(prev), next); + + head = rcu_dereference_protected(master->slaves, + lockdep_is_held(&master->slaves_lock)); + if (q == head) { + rcu_assign_pointer(master->slaves, next); + if (q == next) { + txq = netdev_get_tx_queue(master->dev, 0); + rcu_assign_pointer(master->slaves, NULL); + reset_master_queue = true; + } + } + skb_queue_purge(&dat->q); + break; + } while (prev != rcu_dereference_protected(master->slaves, + lockdep_is_held(&master->slaves_lock))); } + spin_unlock_bh(&master->slaves_lock); + + if (reset_master_queue) + dev_reset_queue(master->dev, txq, NULL); } static int teql_qdisc_init(struct Qdisc *sch, struct nlattr *opt, @@ -168,6 +186,7 @@ static int teql_qdisc_init(struct Qdisc *sch, struct nlattr *opt, struct net_device *dev = qdisc_dev(sch); struct teql_master *m = (struct teql_master *)sch->ops; struct teql_sched_data *q = qdisc_priv(sch); + struct Qdisc *first; if (dev->hard_header_len > m->dev->hard_header_len) return -EINVAL; @@ -184,7 +203,9 @@ static int teql_qdisc_init(struct Qdisc *sch, struct nlattr *opt, skb_queue_head_init(&q->q); - if (m->slaves) { + spin_lock_bh(&m->slaves_lock); + first = rcu_dereference_protected(m->slaves, lockdep_is_held(&m->slaves_lock)); + if (first) { if (m->dev->flags & IFF_UP) { if ((m->dev->flags & IFF_POINTOPOINT && !(dev->flags & IFF_POINTOPOINT)) || @@ -192,8 +213,10 @@ static int teql_qdisc_init(struct Qdisc *sch, struct nlattr *opt, !(dev->flags & IFF_BROADCAST)) || (m->dev->flags & IFF_MULTICAST && !(dev->flags & IFF_MULTICAST)) || - dev->mtu < m->dev->mtu) + dev->mtu < m->dev->mtu) { + spin_unlock_bh(&m->slaves_lock); return -EINVAL; + } } else { if (!(dev->flags&IFF_POINTOPOINT)) m->dev->flags &= ~IFF_POINTOPOINT; @@ -204,14 +227,17 @@ static int teql_qdisc_init(struct Qdisc *sch, struct nlattr *opt, if (dev->mtu < m->dev->mtu) m->dev->mtu = dev->mtu; } - q->next = NEXT_SLAVE(m->slaves); - NEXT_SLAVE(m->slaves) = sch; + rcu_assign_pointer(q->next, + rcu_dereference_protected(NEXT_SLAVE(first), + lockdep_is_held(&m->slaves_lock))); + rcu_assign_pointer(NEXT_SLAVE(first), sch); } else { - q->next = sch; - m->slaves = sch; + rcu_assign_pointer(q->next, sch); + rcu_assign_pointer(m->slaves, sch); m->dev->mtu = dev->mtu; m->dev->flags = (m->dev->flags&~FMASK)|(dev->flags&FMASK); } + spin_unlock_bh(&m->slaves_lock); return 0; } @@ -285,12 +311,14 @@ static netdev_tx_t teql_master_xmit(struct sk_buff *skb, struct net_device *dev) int subq = skb_get_queue_mapping(skb); struct sk_buff *skb_res = NULL; - start = master->slaves; - restart: nores = 0; busy = 0; + rcu_read_lock(); + + start = rcu_dereference(master->slaves); + q = start; if (!q) goto drop; @@ -317,10 +345,17 @@ restart: netdev_start_xmit(skb, slave, slave_txq, false) == NETDEV_TX_OK) { __netif_tx_unlock(slave_txq); - master->slaves = NEXT_SLAVE(q); + spin_lock(&master->slaves_lock); + if (rcu_dereference_protected(master->slaves, + lockdep_is_held(&master->slaves_lock)) == q) + rcu_assign_pointer(master->slaves, + rcu_dereference_protected(NEXT_SLAVE(q), + lockdep_is_held(&master->slaves_lock))); + spin_unlock(&master->slaves_lock); netif_wake_queue(dev); master->tx_packets++; master->tx_bytes += length; + rcu_read_unlock(); return NETDEV_TX_OK; } __netif_tx_unlock(slave_txq); @@ -329,45 +364,56 @@ restart: busy = 1; break; case 1: - master->slaves = NEXT_SLAVE(q); + spin_lock(&master->slaves_lock); + if (rcu_dereference_protected(master->slaves, + lockdep_is_held(&master->slaves_lock)) == q) + rcu_assign_pointer(master->slaves, + rcu_dereference_protected(NEXT_SLAVE(q), + lockdep_is_held(&master->slaves_lock))); + spin_unlock(&master->slaves_lock); + rcu_read_unlock(); return NETDEV_TX_OK; default: nores = 1; break; } __skb_pull(skb, skb_network_offset(skb)); - } while ((q = NEXT_SLAVE(q)) != start); + } while ((q = rcu_dereference(NEXT_SLAVE(q))) != start); if (nores && skb_res == NULL) { skb_res = skb; + rcu_read_unlock(); goto restart; } if (busy) { netif_stop_queue(dev); + rcu_read_unlock(); return NETDEV_TX_BUSY; } master->tx_errors++; drop: master->tx_dropped++; + rcu_read_unlock(); dev_kfree_skb(skb); return NETDEV_TX_OK; } static int teql_master_open(struct net_device *dev) { - struct Qdisc *q; + struct Qdisc *q, *first; struct teql_master *m = netdev_priv(dev); int mtu = 0xFFFE; unsigned int flags = IFF_NOARP | IFF_MULTICAST; - if (m->slaves == NULL) + first = rtnl_dereference(m->slaves); + if (!first) return -EUNATCH; flags = FMASK; - q = m->slaves; + q = first; do { struct net_device *slave = qdisc_dev(q); @@ -389,7 +435,7 @@ static int teql_master_open(struct net_device *dev) flags &= ~IFF_BROADCAST; if (!(slave->flags&IFF_MULTICAST)) flags &= ~IFF_MULTICAST; - } while ((q = NEXT_SLAVE(q)) != m->slaves); + } while ((q = rtnl_dereference(NEXT_SLAVE(q))) != first); m->dev->mtu = mtu; m->dev->flags = (m->dev->flags&~FMASK) | flags; @@ -417,14 +463,15 @@ static void teql_master_stats64(struct net_device *dev, static int teql_master_mtu(struct net_device *dev, int new_mtu) { struct teql_master *m = netdev_priv(dev); - struct Qdisc *q; + struct Qdisc *q, *first; - q = m->slaves; + first = rtnl_dereference(m->slaves); + q = first; if (q) { do { if (new_mtu > qdisc_dev(q)->mtu) return -EINVAL; - } while ((q = NEXT_SLAVE(q)) != m->slaves); + } while ((q = rtnl_dereference(NEXT_SLAVE(q))) != first); } WRITE_ONCE(dev->mtu, new_mtu); @@ -444,6 +491,7 @@ static __init void teql_master_setup(struct net_device *dev) struct teql_master *master = netdev_priv(dev); struct Qdisc_ops *ops = &master->qops; + spin_lock_init(&master->slaves_lock); master->dev = dev; ops->priv_size = sizeof(struct teql_sched_data); |