diff options
Diffstat (limited to 'kernel/arch/arm/net')
| -rw-r--r-- | kernel/arch/arm/net/bpf_jit_32.c | 170 | ||||
| -rw-r--r-- | kernel/arch/arm/net/bpf_jit_32.h | 8 |
2 files changed, 139 insertions, 39 deletions
diff --git a/kernel/arch/arm/net/bpf_jit_32.c b/kernel/arch/arm/net/bpf_jit_32.c index e0e23582c..93d0b6d0b 100644 --- a/kernel/arch/arm/net/bpf_jit_32.c +++ b/kernel/arch/arm/net/bpf_jit_32.c @@ -74,38 +74,58 @@ struct jit_ctx { int bpf_jit_enable __read_mostly; -static u64 jit_get_skb_b(struct sk_buff *skb, unsigned offset) +static inline int call_neg_helper(struct sk_buff *skb, int offset, void *ret, + unsigned int size) +{ + void *ptr = bpf_internal_load_pointer_neg_helper(skb, offset, size); + + if (!ptr) + return -EFAULT; + memcpy(ret, ptr, size); + return 0; +} + +static u64 jit_get_skb_b(struct sk_buff *skb, int offset) { u8 ret; int err; - err = skb_copy_bits(skb, offset, &ret, 1); + if (offset < 0) + err = call_neg_helper(skb, offset, &ret, 1); + else + err = skb_copy_bits(skb, offset, &ret, 1); return (u64)err << 32 | ret; } -static u64 jit_get_skb_h(struct sk_buff *skb, unsigned offset) +static u64 jit_get_skb_h(struct sk_buff *skb, int offset) { u16 ret; int err; - err = skb_copy_bits(skb, offset, &ret, 2); + if (offset < 0) + err = call_neg_helper(skb, offset, &ret, 2); + else + err = skb_copy_bits(skb, offset, &ret, 2); return (u64)err << 32 | ntohs(ret); } -static u64 jit_get_skb_w(struct sk_buff *skb, unsigned offset) +static u64 jit_get_skb_w(struct sk_buff *skb, int offset) { u32 ret; int err; - err = skb_copy_bits(skb, offset, &ret, 4); + if (offset < 0) + err = call_neg_helper(skb, offset, &ret, 4); + else + err = skb_copy_bits(skb, offset, &ret, 4); return (u64)err << 32 | ntohl(ret); } /* - * Wrapper that handles both OABI and EABI and assures Thumb2 interworking + * Wrappers which handle both OABI and EABI and assures Thumb2 interworking * (where the assembly routines like __aeabi_uidiv could cause problems). */ static u32 jit_udiv(u32 dividend, u32 divisor) @@ -113,6 +133,11 @@ static u32 jit_udiv(u32 dividend, u32 divisor) return dividend / divisor; } +static u32 jit_mod(u32 dividend, u32 divisor) +{ + return dividend % divisor; +} + static inline void _emit(int cond, u32 inst, struct jit_ctx *ctx) { inst |= (cond << 28); @@ -162,19 +187,6 @@ static inline int mem_words_used(struct jit_ctx *ctx) return fls(ctx->seen & SEEN_MEM); } -static inline bool is_load_to_a(u16 inst) -{ - switch (inst) { - case BPF_LD | BPF_W | BPF_LEN: - case BPF_LD | BPF_W | BPF_ABS: - case BPF_LD | BPF_H | BPF_ABS: - case BPF_LD | BPF_B | BPF_ABS: - return true; - default: - return false; - } -} - static void jit_fill_hole(void *area, unsigned int size) { u32 *ptr; @@ -186,7 +198,6 @@ static void jit_fill_hole(void *area, unsigned int size) static void build_prologue(struct jit_ctx *ctx) { u16 reg_set = saved_regs(ctx); - u16 first_inst = ctx->skf->insns[0].code; u16 off; #ifdef CONFIG_FRAME_POINTER @@ -216,7 +227,7 @@ static void build_prologue(struct jit_ctx *ctx) emit(ARM_MOV_I(r_X, 0), ctx); /* do not leak kernel data to userspace */ - if ((first_inst != (BPF_RET | BPF_K)) && !(is_load_to_a(first_inst))) + if (bpf_needs_clear_a(&ctx->skf->insns[0])) emit(ARM_MOV_I(r_A, 0), ctx); /* stack space for the BPF_MEM words */ @@ -451,11 +462,17 @@ static inline void emit_blx_r(u8 tgt_reg, struct jit_ctx *ctx) #endif } -static inline void emit_udiv(u8 rd, u8 rm, u8 rn, struct jit_ctx *ctx) +static inline void emit_udivmod(u8 rd, u8 rm, u8 rn, struct jit_ctx *ctx, + int bpf_op) { #if __LINUX_ARM_ARCH__ == 7 if (elf_hwcap & HWCAP_IDIVA) { - emit(ARM_UDIV(rd, rm, rn), ctx); + if (bpf_op == BPF_DIV) + emit(ARM_UDIV(rd, rm, rn), ctx); + else { + emit(ARM_UDIV(ARM_R3, rm, rn), ctx); + emit(ARM_MLS(rd, rn, ARM_R3, rm), ctx); + } return; } #endif @@ -476,7 +493,8 @@ static inline void emit_udiv(u8 rd, u8 rm, u8 rn, struct jit_ctx *ctx) emit(ARM_MOV_R(ARM_R0, rm), ctx); ctx->seen |= SEEN_CALL; - emit_mov_i(ARM_R3, (u32)jit_udiv, ctx); + emit_mov_i(ARM_R3, bpf_op == BPF_DIV ? (u32)jit_udiv : (u32)jit_mod, + ctx); emit_blx_r(ARM_R3, ctx); if (rd != ARM_R0) @@ -536,9 +554,6 @@ static int build_body(struct jit_ctx *ctx) case BPF_LD | BPF_B | BPF_ABS: load_order = 0; load: - /* the interpreter will deal with the negative K */ - if ((int)k < 0) - return -ENOTSUPP; emit_mov_i(r_off, k, ctx); load_common: ctx->seen |= SEEN_DATA | SEEN_CALL; @@ -547,12 +562,24 @@ load_common: emit(ARM_SUB_I(r_scratch, r_skb_hl, 1 << load_order), ctx); emit(ARM_CMP_R(r_scratch, r_off), ctx); - condt = ARM_COND_HS; + condt = ARM_COND_GE; } else { emit(ARM_CMP_R(r_skb_hl, r_off), ctx); condt = ARM_COND_HI; } + /* + * test for negative offset, only if we are + * currently scheduled to take the fast + * path. this will update the flags so that + * the slowpath instruction are ignored if the + * offset is negative. + * + * for loard_order == 0 the HI condition will + * make loads at offset 0 take the slow path too. + */ + _emit(condt, ARM_CMP_I(r_off, 0), ctx); + _emit(condt, ARM_ADD_R(r_scratch, r_off, r_skb_data), ctx); @@ -585,6 +612,7 @@ load_common: case BPF_LD | BPF_B | BPF_IND: load_order = 0; load_ind: + update_on_xread(ctx); OP_IMM3(ARM_ADD, r_off, r_X, k, ctx); goto load_common; case BPF_LDX | BPF_IMM: @@ -668,13 +696,27 @@ load_ind: if (k == 1) break; emit_mov_i(r_scratch, k, ctx); - emit_udiv(r_A, r_A, r_scratch, ctx); + emit_udivmod(r_A, r_A, r_scratch, ctx, BPF_DIV); break; case BPF_ALU | BPF_DIV | BPF_X: update_on_xread(ctx); emit(ARM_CMP_I(r_X, 0), ctx); emit_err_ret(ARM_COND_EQ, ctx); - emit_udiv(r_A, r_A, r_X, ctx); + emit_udivmod(r_A, r_A, r_X, ctx, BPF_DIV); + break; + case BPF_ALU | BPF_MOD | BPF_K: + if (k == 1) { + emit_mov_i(r_A, 0, ctx); + break; + } + emit_mov_i(r_scratch, k, ctx); + emit_udivmod(r_A, r_A, r_scratch, ctx, BPF_MOD); + break; + case BPF_ALU | BPF_MOD | BPF_X: + update_on_xread(ctx); + emit(ARM_CMP_I(r_X, 0), ctx); + emit_err_ret(ARM_COND_EQ, ctx); + emit_udivmod(r_A, r_A, r_X, ctx, BPF_MOD); break; case BPF_ALU | BPF_OR | BPF_K: /* A |= K */ @@ -714,7 +756,8 @@ load_ind: case BPF_ALU | BPF_RSH | BPF_K: if (unlikely(k > 31)) return -1; - emit(ARM_LSR_I(r_A, r_A, k), ctx); + if (k) + emit(ARM_LSR_I(r_A, r_A, k), ctx); break; case BPF_ALU | BPF_RSH | BPF_X: update_on_xread(ctx); @@ -828,7 +871,9 @@ b_epilogue: emit(ARM_LDR_I(r_A, r_scratch, off), ctx); break; case BPF_ANC | SKF_AD_IFINDEX: + case BPF_ANC | SKF_AD_HATYPE: /* A = skb->dev->ifindex */ + /* A = skb->dev->type */ ctx->seen |= SEEN_SKB; off = offsetof(struct sk_buff, dev); emit(ARM_LDR_I(r_scratch, r_skb, off), ctx); @@ -838,8 +883,24 @@ b_epilogue: BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, ifindex) != 4); - off = offsetof(struct net_device, ifindex); - emit(ARM_LDR_I(r_A, r_scratch, off), ctx); + BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, + type) != 2); + + if (code == (BPF_ANC | SKF_AD_IFINDEX)) { + off = offsetof(struct net_device, ifindex); + emit(ARM_LDR_I(r_A, r_scratch, off), ctx); + } else { + /* + * offset of field "type" in "struct + * net_device" is above what can be + * used in the ldrh rd, [rn, #imm] + * instruction, so load the offset in + * a register and use ldrh rd, [rn, rm] + */ + off = offsetof(struct net_device, type); + emit_mov_i(ARM_R3, off, ctx); + emit(ARM_LDRH_R(r_A, r_scratch, ARM_R3), ctx); + } break; case BPF_ANC | SKF_AD_MARK: ctx->seen |= SEEN_SKB; @@ -860,9 +921,22 @@ b_epilogue: off = offsetof(struct sk_buff, vlan_tci); emit(ARM_LDRH_I(r_A, r_skb, off), ctx); if (code == (BPF_ANC | SKF_AD_VLAN_TAG)) - OP_IMM3(ARM_AND, r_A, r_A, VLAN_VID_MASK, ctx); - else - OP_IMM3(ARM_AND, r_A, r_A, VLAN_TAG_PRESENT, ctx); + OP_IMM3(ARM_AND, r_A, r_A, ~VLAN_TAG_PRESENT, ctx); + else { + OP_IMM3(ARM_LSR, r_A, r_A, 12, ctx); + OP_IMM3(ARM_AND, r_A, r_A, 0x1, ctx); + } + break; + case BPF_ANC | SKF_AD_PKTTYPE: + ctx->seen |= SEEN_SKB; + BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, + __pkt_type_offset[0]) != 1); + off = PKT_TYPE_OFFSET(); + emit(ARM_LDRB_I(r_A, r_skb, off), ctx); + emit(ARM_AND_I(r_A, r_A, PKT_TYPE_MAX), ctx); +#ifdef __BIG_ENDIAN_BITFIELD + emit(ARM_LSR_I(r_A, r_A, 5), ctx); +#endif break; case BPF_ANC | SKF_AD_QUEUE: ctx->seen |= SEEN_SKB; @@ -873,6 +947,24 @@ b_epilogue: off = offsetof(struct sk_buff, queue_mapping); emit(ARM_LDRH_I(r_A, r_skb, off), ctx); break; + case BPF_ANC | SKF_AD_PAY_OFFSET: + ctx->seen |= SEEN_SKB | SEEN_CALL; + + emit(ARM_MOV_R(ARM_R0, r_skb), ctx); + emit_mov_i(ARM_R3, (unsigned int)skb_get_poff, ctx); + emit_blx_r(ARM_R3, ctx); + emit(ARM_MOV_R(r_A, ARM_R0), ctx); + break; + case BPF_LDX | BPF_W | BPF_ABS: + /* + * load a 32bit word from struct seccomp_data. + * seccomp_check_filter() will already have checked + * that k is 32bit aligned and lies within the + * struct seccomp_data. + */ + ctx->seen |= SEEN_SKB; + emit(ARM_LDR_I(r_A, r_skb, k), ctx); + break; default: return -1; } @@ -956,7 +1048,7 @@ void bpf_jit_compile(struct bpf_prog *fp) } build_epilogue(&ctx); - flush_icache_range((u32)ctx.target, (u32)(ctx.target + ctx.idx)); + flush_icache_range((u32)header, (u32)(ctx.target + ctx.idx)); #if __LINUX_ARM_ARCH__ < 7 if (ctx.imm_count) @@ -969,7 +1061,7 @@ void bpf_jit_compile(struct bpf_prog *fp) set_memory_ro((unsigned long)header, header->pages); fp->bpf_func = (void *)ctx.target; - fp->jited = true; + fp->jited = 1; out: kfree(ctx.offsets); return; diff --git a/kernel/arch/arm/net/bpf_jit_32.h b/kernel/arch/arm/net/bpf_jit_32.h index b2d7d9285..c46fca297 100644 --- a/kernel/arch/arm/net/bpf_jit_32.h +++ b/kernel/arch/arm/net/bpf_jit_32.h @@ -74,6 +74,7 @@ #define ARM_INST_LDRB_I 0x05d00000 #define ARM_INST_LDRB_R 0x07d00000 #define ARM_INST_LDRH_I 0x01d000b0 +#define ARM_INST_LDRH_R 0x019000b0 #define ARM_INST_LDR_I 0x05900000 #define ARM_INST_LDM 0x08900000 @@ -114,6 +115,8 @@ #define ARM_INST_UMULL 0x00800090 +#define ARM_INST_MLS 0x00600090 + /* * Use a suitable undefined instruction to use for ARM/Thumb2 faulting. * We need to be careful not to conflict with those used by other modules @@ -160,6 +163,8 @@ | (rm)) #define ARM_LDRH_I(rt, rn, off) (ARM_INST_LDRH_I | (rt) << 12 | (rn) << 16 \ | (((off) & 0xf0) << 4) | ((off) & 0xf)) +#define ARM_LDRH_R(rt, rn, rm) (ARM_INST_LDRH_R | (rt) << 12 | (rn) << 16 \ + | (rm)) #define ARM_LDM(rn, regs) (ARM_INST_LDM | (rn) << 16 | (regs)) @@ -207,4 +212,7 @@ #define ARM_UMULL(rd_lo, rd_hi, rn, rm) (ARM_INST_UMULL | (rd_hi) << 16 \ | (rd_lo) << 12 | (rm) << 8 | rn) +#define ARM_MLS(rd, rn, rm, ra) (ARM_INST_MLS | (rd) << 16 | (rn) | (rm) << 8 \ + | (ra) << 12) + #endif /* PFILTER_OPCODES_ARM_H */ |