summaryrefslogtreecommitdiffstats
path: root/kernel/arch/arm/net
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/arch/arm/net')
-rw-r--r--kernel/arch/arm/net/bpf_jit_32.c170
-rw-r--r--kernel/arch/arm/net/bpf_jit_32.h8
2 files changed, 139 insertions, 39 deletions
diff --git a/kernel/arch/arm/net/bpf_jit_32.c b/kernel/arch/arm/net/bpf_jit_32.c
index e0e23582c..93d0b6d0b 100644
--- a/kernel/arch/arm/net/bpf_jit_32.c
+++ b/kernel/arch/arm/net/bpf_jit_32.c
@@ -74,38 +74,58 @@ struct jit_ctx {
int bpf_jit_enable __read_mostly;
-static u64 jit_get_skb_b(struct sk_buff *skb, unsigned offset)
+static inline int call_neg_helper(struct sk_buff *skb, int offset, void *ret,
+ unsigned int size)
+{
+ void *ptr = bpf_internal_load_pointer_neg_helper(skb, offset, size);
+
+ if (!ptr)
+ return -EFAULT;
+ memcpy(ret, ptr, size);
+ return 0;
+}
+
+static u64 jit_get_skb_b(struct sk_buff *skb, int offset)
{
u8 ret;
int err;
- err = skb_copy_bits(skb, offset, &ret, 1);
+ if (offset < 0)
+ err = call_neg_helper(skb, offset, &ret, 1);
+ else
+ err = skb_copy_bits(skb, offset, &ret, 1);
return (u64)err << 32 | ret;
}
-static u64 jit_get_skb_h(struct sk_buff *skb, unsigned offset)
+static u64 jit_get_skb_h(struct sk_buff *skb, int offset)
{
u16 ret;
int err;
- err = skb_copy_bits(skb, offset, &ret, 2);
+ if (offset < 0)
+ err = call_neg_helper(skb, offset, &ret, 2);
+ else
+ err = skb_copy_bits(skb, offset, &ret, 2);
return (u64)err << 32 | ntohs(ret);
}
-static u64 jit_get_skb_w(struct sk_buff *skb, unsigned offset)
+static u64 jit_get_skb_w(struct sk_buff *skb, int offset)
{
u32 ret;
int err;
- err = skb_copy_bits(skb, offset, &ret, 4);
+ if (offset < 0)
+ err = call_neg_helper(skb, offset, &ret, 4);
+ else
+ err = skb_copy_bits(skb, offset, &ret, 4);
return (u64)err << 32 | ntohl(ret);
}
/*
- * Wrapper that handles both OABI and EABI and assures Thumb2 interworking
+ * Wrappers which handle both OABI and EABI and assures Thumb2 interworking
* (where the assembly routines like __aeabi_uidiv could cause problems).
*/
static u32 jit_udiv(u32 dividend, u32 divisor)
@@ -113,6 +133,11 @@ static u32 jit_udiv(u32 dividend, u32 divisor)
return dividend / divisor;
}
+static u32 jit_mod(u32 dividend, u32 divisor)
+{
+ return dividend % divisor;
+}
+
static inline void _emit(int cond, u32 inst, struct jit_ctx *ctx)
{
inst |= (cond << 28);
@@ -162,19 +187,6 @@ static inline int mem_words_used(struct jit_ctx *ctx)
return fls(ctx->seen & SEEN_MEM);
}
-static inline bool is_load_to_a(u16 inst)
-{
- switch (inst) {
- case BPF_LD | BPF_W | BPF_LEN:
- case BPF_LD | BPF_W | BPF_ABS:
- case BPF_LD | BPF_H | BPF_ABS:
- case BPF_LD | BPF_B | BPF_ABS:
- return true;
- default:
- return false;
- }
-}
-
static void jit_fill_hole(void *area, unsigned int size)
{
u32 *ptr;
@@ -186,7 +198,6 @@ static void jit_fill_hole(void *area, unsigned int size)
static void build_prologue(struct jit_ctx *ctx)
{
u16 reg_set = saved_regs(ctx);
- u16 first_inst = ctx->skf->insns[0].code;
u16 off;
#ifdef CONFIG_FRAME_POINTER
@@ -216,7 +227,7 @@ static void build_prologue(struct jit_ctx *ctx)
emit(ARM_MOV_I(r_X, 0), ctx);
/* do not leak kernel data to userspace */
- if ((first_inst != (BPF_RET | BPF_K)) && !(is_load_to_a(first_inst)))
+ if (bpf_needs_clear_a(&ctx->skf->insns[0]))
emit(ARM_MOV_I(r_A, 0), ctx);
/* stack space for the BPF_MEM words */
@@ -451,11 +462,17 @@ static inline void emit_blx_r(u8 tgt_reg, struct jit_ctx *ctx)
#endif
}
-static inline void emit_udiv(u8 rd, u8 rm, u8 rn, struct jit_ctx *ctx)
+static inline void emit_udivmod(u8 rd, u8 rm, u8 rn, struct jit_ctx *ctx,
+ int bpf_op)
{
#if __LINUX_ARM_ARCH__ == 7
if (elf_hwcap & HWCAP_IDIVA) {
- emit(ARM_UDIV(rd, rm, rn), ctx);
+ if (bpf_op == BPF_DIV)
+ emit(ARM_UDIV(rd, rm, rn), ctx);
+ else {
+ emit(ARM_UDIV(ARM_R3, rm, rn), ctx);
+ emit(ARM_MLS(rd, rn, ARM_R3, rm), ctx);
+ }
return;
}
#endif
@@ -476,7 +493,8 @@ static inline void emit_udiv(u8 rd, u8 rm, u8 rn, struct jit_ctx *ctx)
emit(ARM_MOV_R(ARM_R0, rm), ctx);
ctx->seen |= SEEN_CALL;
- emit_mov_i(ARM_R3, (u32)jit_udiv, ctx);
+ emit_mov_i(ARM_R3, bpf_op == BPF_DIV ? (u32)jit_udiv : (u32)jit_mod,
+ ctx);
emit_blx_r(ARM_R3, ctx);
if (rd != ARM_R0)
@@ -536,9 +554,6 @@ static int build_body(struct jit_ctx *ctx)
case BPF_LD | BPF_B | BPF_ABS:
load_order = 0;
load:
- /* the interpreter will deal with the negative K */
- if ((int)k < 0)
- return -ENOTSUPP;
emit_mov_i(r_off, k, ctx);
load_common:
ctx->seen |= SEEN_DATA | SEEN_CALL;
@@ -547,12 +562,24 @@ load_common:
emit(ARM_SUB_I(r_scratch, r_skb_hl,
1 << load_order), ctx);
emit(ARM_CMP_R(r_scratch, r_off), ctx);
- condt = ARM_COND_HS;
+ condt = ARM_COND_GE;
} else {
emit(ARM_CMP_R(r_skb_hl, r_off), ctx);
condt = ARM_COND_HI;
}
+ /*
+ * test for negative offset, only if we are
+ * currently scheduled to take the fast
+ * path. this will update the flags so that
+ * the slowpath instruction are ignored if the
+ * offset is negative.
+ *
+ * for loard_order == 0 the HI condition will
+ * make loads at offset 0 take the slow path too.
+ */
+ _emit(condt, ARM_CMP_I(r_off, 0), ctx);
+
_emit(condt, ARM_ADD_R(r_scratch, r_off, r_skb_data),
ctx);
@@ -585,6 +612,7 @@ load_common:
case BPF_LD | BPF_B | BPF_IND:
load_order = 0;
load_ind:
+ update_on_xread(ctx);
OP_IMM3(ARM_ADD, r_off, r_X, k, ctx);
goto load_common;
case BPF_LDX | BPF_IMM:
@@ -668,13 +696,27 @@ load_ind:
if (k == 1)
break;
emit_mov_i(r_scratch, k, ctx);
- emit_udiv(r_A, r_A, r_scratch, ctx);
+ emit_udivmod(r_A, r_A, r_scratch, ctx, BPF_DIV);
break;
case BPF_ALU | BPF_DIV | BPF_X:
update_on_xread(ctx);
emit(ARM_CMP_I(r_X, 0), ctx);
emit_err_ret(ARM_COND_EQ, ctx);
- emit_udiv(r_A, r_A, r_X, ctx);
+ emit_udivmod(r_A, r_A, r_X, ctx, BPF_DIV);
+ break;
+ case BPF_ALU | BPF_MOD | BPF_K:
+ if (k == 1) {
+ emit_mov_i(r_A, 0, ctx);
+ break;
+ }
+ emit_mov_i(r_scratch, k, ctx);
+ emit_udivmod(r_A, r_A, r_scratch, ctx, BPF_MOD);
+ break;
+ case BPF_ALU | BPF_MOD | BPF_X:
+ update_on_xread(ctx);
+ emit(ARM_CMP_I(r_X, 0), ctx);
+ emit_err_ret(ARM_COND_EQ, ctx);
+ emit_udivmod(r_A, r_A, r_X, ctx, BPF_MOD);
break;
case BPF_ALU | BPF_OR | BPF_K:
/* A |= K */
@@ -714,7 +756,8 @@ load_ind:
case BPF_ALU | BPF_RSH | BPF_K:
if (unlikely(k > 31))
return -1;
- emit(ARM_LSR_I(r_A, r_A, k), ctx);
+ if (k)
+ emit(ARM_LSR_I(r_A, r_A, k), ctx);
break;
case BPF_ALU | BPF_RSH | BPF_X:
update_on_xread(ctx);
@@ -828,7 +871,9 @@ b_epilogue:
emit(ARM_LDR_I(r_A, r_scratch, off), ctx);
break;
case BPF_ANC | SKF_AD_IFINDEX:
+ case BPF_ANC | SKF_AD_HATYPE:
/* A = skb->dev->ifindex */
+ /* A = skb->dev->type */
ctx->seen |= SEEN_SKB;
off = offsetof(struct sk_buff, dev);
emit(ARM_LDR_I(r_scratch, r_skb, off), ctx);
@@ -838,8 +883,24 @@ b_epilogue:
BUILD_BUG_ON(FIELD_SIZEOF(struct net_device,
ifindex) != 4);
- off = offsetof(struct net_device, ifindex);
- emit(ARM_LDR_I(r_A, r_scratch, off), ctx);
+ BUILD_BUG_ON(FIELD_SIZEOF(struct net_device,
+ type) != 2);
+
+ if (code == (BPF_ANC | SKF_AD_IFINDEX)) {
+ off = offsetof(struct net_device, ifindex);
+ emit(ARM_LDR_I(r_A, r_scratch, off), ctx);
+ } else {
+ /*
+ * offset of field "type" in "struct
+ * net_device" is above what can be
+ * used in the ldrh rd, [rn, #imm]
+ * instruction, so load the offset in
+ * a register and use ldrh rd, [rn, rm]
+ */
+ off = offsetof(struct net_device, type);
+ emit_mov_i(ARM_R3, off, ctx);
+ emit(ARM_LDRH_R(r_A, r_scratch, ARM_R3), ctx);
+ }
break;
case BPF_ANC | SKF_AD_MARK:
ctx->seen |= SEEN_SKB;
@@ -860,9 +921,22 @@ b_epilogue:
off = offsetof(struct sk_buff, vlan_tci);
emit(ARM_LDRH_I(r_A, r_skb, off), ctx);
if (code == (BPF_ANC | SKF_AD_VLAN_TAG))
- OP_IMM3(ARM_AND, r_A, r_A, VLAN_VID_MASK, ctx);
- else
- OP_IMM3(ARM_AND, r_A, r_A, VLAN_TAG_PRESENT, ctx);
+ OP_IMM3(ARM_AND, r_A, r_A, ~VLAN_TAG_PRESENT, ctx);
+ else {
+ OP_IMM3(ARM_LSR, r_A, r_A, 12, ctx);
+ OP_IMM3(ARM_AND, r_A, r_A, 0x1, ctx);
+ }
+ break;
+ case BPF_ANC | SKF_AD_PKTTYPE:
+ ctx->seen |= SEEN_SKB;
+ BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff,
+ __pkt_type_offset[0]) != 1);
+ off = PKT_TYPE_OFFSET();
+ emit(ARM_LDRB_I(r_A, r_skb, off), ctx);
+ emit(ARM_AND_I(r_A, r_A, PKT_TYPE_MAX), ctx);
+#ifdef __BIG_ENDIAN_BITFIELD
+ emit(ARM_LSR_I(r_A, r_A, 5), ctx);
+#endif
break;
case BPF_ANC | SKF_AD_QUEUE:
ctx->seen |= SEEN_SKB;
@@ -873,6 +947,24 @@ b_epilogue:
off = offsetof(struct sk_buff, queue_mapping);
emit(ARM_LDRH_I(r_A, r_skb, off), ctx);
break;
+ case BPF_ANC | SKF_AD_PAY_OFFSET:
+ ctx->seen |= SEEN_SKB | SEEN_CALL;
+
+ emit(ARM_MOV_R(ARM_R0, r_skb), ctx);
+ emit_mov_i(ARM_R3, (unsigned int)skb_get_poff, ctx);
+ emit_blx_r(ARM_R3, ctx);
+ emit(ARM_MOV_R(r_A, ARM_R0), ctx);
+ break;
+ case BPF_LDX | BPF_W | BPF_ABS:
+ /*
+ * load a 32bit word from struct seccomp_data.
+ * seccomp_check_filter() will already have checked
+ * that k is 32bit aligned and lies within the
+ * struct seccomp_data.
+ */
+ ctx->seen |= SEEN_SKB;
+ emit(ARM_LDR_I(r_A, r_skb, k), ctx);
+ break;
default:
return -1;
}
@@ -956,7 +1048,7 @@ void bpf_jit_compile(struct bpf_prog *fp)
}
build_epilogue(&ctx);
- flush_icache_range((u32)ctx.target, (u32)(ctx.target + ctx.idx));
+ flush_icache_range((u32)header, (u32)(ctx.target + ctx.idx));
#if __LINUX_ARM_ARCH__ < 7
if (ctx.imm_count)
@@ -969,7 +1061,7 @@ void bpf_jit_compile(struct bpf_prog *fp)
set_memory_ro((unsigned long)header, header->pages);
fp->bpf_func = (void *)ctx.target;
- fp->jited = true;
+ fp->jited = 1;
out:
kfree(ctx.offsets);
return;
diff --git a/kernel/arch/arm/net/bpf_jit_32.h b/kernel/arch/arm/net/bpf_jit_32.h
index b2d7d9285..c46fca297 100644
--- a/kernel/arch/arm/net/bpf_jit_32.h
+++ b/kernel/arch/arm/net/bpf_jit_32.h
@@ -74,6 +74,7 @@
#define ARM_INST_LDRB_I 0x05d00000
#define ARM_INST_LDRB_R 0x07d00000
#define ARM_INST_LDRH_I 0x01d000b0
+#define ARM_INST_LDRH_R 0x019000b0
#define ARM_INST_LDR_I 0x05900000
#define ARM_INST_LDM 0x08900000
@@ -114,6 +115,8 @@
#define ARM_INST_UMULL 0x00800090
+#define ARM_INST_MLS 0x00600090
+
/*
* Use a suitable undefined instruction to use for ARM/Thumb2 faulting.
* We need to be careful not to conflict with those used by other modules
@@ -160,6 +163,8 @@
| (rm))
#define ARM_LDRH_I(rt, rn, off) (ARM_INST_LDRH_I | (rt) << 12 | (rn) << 16 \
| (((off) & 0xf0) << 4) | ((off) & 0xf))
+#define ARM_LDRH_R(rt, rn, rm) (ARM_INST_LDRH_R | (rt) << 12 | (rn) << 16 \
+ | (rm))
#define ARM_LDM(rn, regs) (ARM_INST_LDM | (rn) << 16 | (regs))
@@ -207,4 +212,7 @@
#define ARM_UMULL(rd_lo, rd_hi, rn, rm) (ARM_INST_UMULL | (rd_hi) << 16 \
| (rd_lo) << 12 | (rm) << 8 | rn)
+#define ARM_MLS(rd, rn, rm, ra) (ARM_INST_MLS | (rd) << 16 | (rn) | (rm) << 8 \
+ | (ra) << 12)
+
#endif /* PFILTER_OPCODES_ARM_H */