diff options
Diffstat (limited to 'framework/src/suricata/src')
70 files changed, 2946 insertions, 1443 deletions
diff --git a/framework/src/suricata/src/Makefile.am b/framework/src/suricata/src/Makefile.am index 674043f5..98e094c1 100644 --- a/framework/src/suricata/src/Makefile.am +++ b/framework/src/suricata/src/Makefile.am @@ -356,6 +356,7 @@ util-lua-common.c util-lua-common.h \ util-lua-dns.c util-lua-dns.h \ util-lua-http.c util-lua-http.h \ util-lua-tls.c util-lua-tls.h \ +util-lua-ssh.c util-lua-ssh.h \ util-magic.c util-magic.h \ util-memcmp.c util-memcmp.h \ util-memcpy.h \ diff --git a/framework/src/suricata/src/alert-unified2-alert.c b/framework/src/suricata/src/alert-unified2-alert.c index ede624c4..facc66b2 100644 --- a/framework/src/suricata/src/alert-unified2-alert.c +++ b/framework/src/suricata/src/alert-unified2-alert.c @@ -186,8 +186,11 @@ typedef struct AlertUnified2Packet_ { typedef struct Unified2AlertFileCtx_ { LogFileCtx *file_ctx; HttpXFFCfg *xff_cfg; + uint32_t flags; /**< flags for all alerts */ } Unified2AlertFileCtx; +#define UNIFIED2_ALERT_FLAGS_EMIT_PACKET (1 << 0) + /** * Unified2 thread vars * @@ -698,6 +701,9 @@ static int Unified2PacketTypeAlert(Unified2AlertThread *aun, const Packet *p, ui { int ret = 0; + if (!(aun->unified2alert_ctx->flags & UNIFIED2_ALERT_FLAGS_EMIT_PACKET)) + return 1; + /* try stream logging first */ if (stream) { SCLogDebug("logging the state"); @@ -1299,6 +1305,20 @@ OutputCtx *Unified2AlertInitCtx(ConfNode *conf) } } + uint32_t flags = UNIFIED2_ALERT_FLAGS_EMIT_PACKET; + if (conf != NULL) { + const char *payload = NULL; + payload = ConfNodeLookupChildValue(conf, "payload"); + if (payload) { + if (ConfValIsFalse(payload)) { + flags &= ~UNIFIED2_ALERT_FLAGS_EMIT_PACKET; + } else if (!ConfValIsTrue(payload)) { + SCLogError(SC_ERR_INVALID_ARGUMENT, "Failed to initialize unified2 output, invalid payload: %s", payload); + exit(EXIT_FAILURE); + } + } + } + ret = Unified2AlertOpenFileCtx(file_ctx, filename); if (ret < 0) goto error; @@ -1325,6 +1345,7 @@ OutputCtx *Unified2AlertInitCtx(ConfNode *conf) unified2alert_ctx->file_ctx = file_ctx; unified2alert_ctx->xff_cfg = xff_cfg; + unified2alert_ctx->flags = flags; output_ctx->data = unified2alert_ctx; output_ctx->DeInit = Unified2AlertDeInitCtx; diff --git a/framework/src/suricata/src/app-layer-htp.c b/framework/src/suricata/src/app-layer-htp.c index 452a607e..0abcda3b 100644 --- a/framework/src/suricata/src/app-layer-htp.c +++ b/framework/src/suricata/src/app-layer-htp.c @@ -78,10 +78,6 @@ #include "util-memcmp.h" -#ifndef HAVE_HTP_SET_PATH_DECODE_U_ENCODING -void htp_config_set_path_decode_u_encoding(htp_cfg_t *cfg, int decode_u_encoding); -#endif - //#define PRINT /** Fast lookup tree (radix) for the various HTP configurations */ diff --git a/framework/src/suricata/src/app-layer-smtp.c b/framework/src/suricata/src/app-layer-smtp.c index 1951613d..0c161edb 100644 --- a/framework/src/suricata/src/app-layer-smtp.c +++ b/framework/src/suricata/src/app-layer-smtp.c @@ -220,7 +220,9 @@ SCEnumCharMap smtp_reply_map[ ] = { }; /* Create SMTP config structure */ -SMTPConfig smtp_config = { 0, { 0, 0, 0, 0 }, 0, 0, 0}; +SMTPConfig smtp_config = { 0, { 0, 0, 0, 0, 0 }, 0, 0, 0}; + +static SMTPString *SMTPStringAlloc(void); /** * \brief Configure SMTP Mime Decoder by parsing out mime section of YAML @@ -264,6 +266,11 @@ static void SMTPConfigure(void) { if (ret) { smtp_config.mime_config.extract_urls = val; } + + ret = ConfGetChildValueBool(config, "body-md5", &val); + if (ret) { + smtp_config.mime_config.body_md5 = val; + } } /* Pass mime config data to MimeDec API */ @@ -323,6 +330,7 @@ static SMTPTransaction *SMTPTransactionCreate(void) return NULL; } + TAILQ_INIT(&tx->rcpt_to_list); tx->mime_state = NULL; return tx; } @@ -366,7 +374,7 @@ int SMTPProcessDataChunk(const uint8_t *chunk, uint32_t len, if (smtp_state->files_ts == NULL) { ret = MIME_DEC_ERR_MEM; SCLogError(SC_ERR_MEM_ALLOC, "Could not create file container"); - goto end; + SCReturnInt(ret); } } files = smtp_state->files_ts; @@ -448,7 +456,7 @@ int SMTPProcessDataChunk(const uint8_t *chunk, uint32_t len, if (files != NULL) { FilePrune(files); } -end: + SCReturnInt(ret); } @@ -805,7 +813,8 @@ static int SMTPProcessCommandDATA(SMTPState *state, Flow *f, if (smtp_config.decode_mime && state->curr_tx->mime_state) { int ret = MimeDecParseLine((const uint8_t *) state->current_line, - state->current_line_len, state->curr_tx->mime_state); + state->current_line_len, state->current_line_delimiter_len, + state->curr_tx->mime_state); if (ret != MIME_DEC_OK) { SCLogDebug("MimeDecParseLine() function returned an error code: %d", ret); } @@ -955,6 +964,71 @@ static int SMTPParseCommandBDAT(SMTPState *state) return 0; } +static int SMTPParseCommandWithParam(SMTPState *state, uint8_t prefix_len, uint8_t **target, uint16_t *target_len) +{ + int i = prefix_len + 1; + int spc_i = 0; + + while (i < state->current_line_len) { + if (state->current_line[i] != ' ') { + break; + } + i++; + } + + /* rfc1870: with the size extension the mail from can be followed by an option. + We use the space separator to detect it. */ + spc_i = i; + while (spc_i < state->current_line_len) { + if (state->current_line[spc_i] == ' ') { + break; + } + spc_i++; + } + + *target = SCMalloc(spc_i - i + 1); + if (*target == NULL) + return -1; + memcpy(*target, state->current_line + i, spc_i - i); + (*target)[spc_i - i] = '\0'; + *target_len = spc_i - i; + + return 0; +} + +static int SMTPParseCommandHELO(SMTPState *state) +{ + return SMTPParseCommandWithParam(state, 4, &state->helo, &state->helo_len); +} + +static int SMTPParseCommandMAILFROM(SMTPState *state) +{ + return SMTPParseCommandWithParam(state, 9, + &state->curr_tx->mail_from, + &state->curr_tx->mail_from_len); +} + +static int SMTPParseCommandRCPTTO(SMTPState *state) +{ + uint8_t *rcptto; + uint16_t rcptto_len; + + if (SMTPParseCommandWithParam(state, 7, &rcptto, &rcptto_len) == 0) { + SMTPString *rcptto_str = SMTPStringAlloc(); + if (rcptto_str) { + rcptto_str->str = rcptto; + rcptto_str->len = rcptto_len; + TAILQ_INSERT_TAIL(&state->curr_tx->rcpt_to_list, rcptto_str, next); + } else { + SCFree(rcptto); + return -1; + } + } else { + return -1; + } + return 0; +} + /* consider 'rset' and 'quit' to be part of the existing state */ static int NoNewTx(SMTPState *state) { @@ -1027,6 +1101,28 @@ static int SMTPProcessRequest(SMTPState *state, Flow *f, } state->current_command = SMTP_COMMAND_BDAT; state->parser_state |= SMTP_PARSER_STATE_COMMAND_DATA_MODE; + } else if (state->current_line_len >= 4 && + ((SCMemcmpLowercase("helo", state->current_line, 4) == 0) || + SCMemcmpLowercase("ehlo", state->current_line, 4) == 0)) { + r = SMTPParseCommandHELO(state); + if (r == -1) { + SCReturnInt(-1); + } + state->current_command = SMTP_COMMAND_OTHER_CMD; + } else if (state->current_line_len >= 9 && + SCMemcmpLowercase("mail from", state->current_line, 9) == 0) { + r = SMTPParseCommandMAILFROM(state); + if (r == -1) { + SCReturnInt(-1); + } + state->current_command = SMTP_COMMAND_OTHER_CMD; + } else if (state->current_line_len >= 7 && + SCMemcmpLowercase("rcpt to", state->current_line, 7) == 0) { + r = SMTPParseCommandRCPTTO(state); + if (r == -1) { + SCReturnInt(-1); + } + state->current_command = SMTP_COMMAND_OTHER_CMD; } else { state->current_command = SMTP_COMMAND_OTHER_CMD; } @@ -1142,6 +1238,25 @@ void *SMTPStateAlloc(void) return smtp_state; } +static SMTPString *SMTPStringAlloc(void) +{ + SMTPString *smtp_string = SCMalloc(sizeof(SMTPString)); + if (unlikely(smtp_string == NULL)) + return NULL; + memset(smtp_string, 0, sizeof(SMTPString)); + + return smtp_string; +} + + +static void SMTPStringFree(SMTPString *str) +{ + if (str->str) { + SCFree(str->str); + } + SCFree(str); +} + static void *SMTPLocalStorageAlloc(void) { /* needed by the mpm */ @@ -1178,6 +1293,15 @@ static void SMTPTransactionFree(SMTPTransaction *tx, SMTPState *state) if (tx->de_state != NULL) DetectEngineStateFree(tx->de_state); + + if (tx->mail_from) + SCFree(tx->mail_from); + + SMTPString *str = NULL; + while ((str = TAILQ_FIRST(&tx->rcpt_to_list))) { + TAILQ_REMOVE(&tx->rcpt_to_list, str, next); + SMTPStringFree(str); + } #if 0 if (tx->decoder_events->cnt <= smtp_state->events) smtp_state->events -= tx->decoder_events->cnt; @@ -1205,6 +1329,10 @@ static void SMTPStateFree(void *p) SCFree(smtp_state->tc_db); } + if (smtp_state->helo) { + SCFree(smtp_state->helo); + } + FileContainerFree(smtp_state->files_ts); SMTPTransaction *tx = NULL; @@ -4365,6 +4493,13 @@ int SMTPParserTest14(void) SCMutexUnlock(&f.m); goto end; } + + if ((smtp_state->helo_len != 7) || strncmp("boo.com", (char *)smtp_state->helo, 7)) { + printf("incorrect parsing of HELO field '%s' (%d)\n", smtp_state->helo, smtp_state->helo_len); + SCMutexUnlock(&f.m); + goto end; + } + SCMutexUnlock(&f.m); if (smtp_state->input_len != 0 || smtp_state->cmds_cnt != 0 || @@ -4402,6 +4537,16 @@ int SMTPParserTest14(void) SCMutexUnlock(&f.m); goto end; } + + if ((smtp_state->curr_tx->mail_from_len != 14) || + strncmp("asdff@asdf.com", (char *)smtp_state->curr_tx->mail_from, 14)) { + printf("incorrect parsing of MAIL FROM field '%s' (%d)\n", + smtp_state->curr_tx->mail_from, + smtp_state->curr_tx->mail_from_len); + SCMutexUnlock(&f.m); + goto end; + } + SCMutexUnlock(&f.m); if (smtp_state->input_len != 0 || smtp_state->cmds_cnt != 0 || diff --git a/framework/src/suricata/src/app-layer-smtp.h b/framework/src/suricata/src/app-layer-smtp.h index 02090c61..c5868414 100644 --- a/framework/src/suricata/src/app-layer-smtp.h +++ b/framework/src/suricata/src/app-layer-smtp.h @@ -51,6 +51,13 @@ enum { SMTP_DECODER_EVENT_MIME_BOUNDARY_TOO_LONG, }; +typedef struct SMTPString_ { + uint8_t *str; + uint16_t len; + + TAILQ_ENTRY(SMTPString_) next; +} SMTPString; + typedef struct SMTPTransaction_ { /** id of this tx, starting at 0 */ uint64_t tx_id; @@ -65,6 +72,12 @@ typedef struct SMTPTransaction_ { AppLayerDecoderEvents *decoder_events; /**< per tx events */ DetectEngineState *de_state; + /* MAIL FROM parameters */ + uint8_t *mail_from; + uint16_t mail_from_len; + + TAILQ_HEAD(, SMTPString_) rcpt_to_list; /**< rcpt to string list */ + TAILQ_ENTRY(SMTPTransaction_) next; } SMTPTransaction; @@ -136,6 +149,9 @@ typedef struct SMTPState_ { /** the list of files sent to the server */ FileContainer *files_ts; + /* HELO of HELO message content */ + uint8_t *helo; + uint16_t helo_len; } SMTPState; /* Create SMTP config structure */ diff --git a/framework/src/suricata/src/app-layer-ssh.h b/framework/src/suricata/src/app-layer-ssh.h index 7a6a9b72..4fd59aa9 100644 --- a/framework/src/suricata/src/app-layer-ssh.h +++ b/framework/src/suricata/src/app-layer-ssh.h @@ -34,6 +34,8 @@ #define SSH_FLAG_STATE_LOGGED 0x04 +#define SSH_FLAG_STATE_LOGGED_LUA 0x08 + /* MSG_CODE */ #define SSH_MSG_NEWKEYS 21 diff --git a/framework/src/suricata/src/app-layer-ssl.h b/framework/src/suricata/src/app-layer-ssl.h index 5c6fe5b1..2fc1a969 100644 --- a/framework/src/suricata/src/app-layer-ssl.h +++ b/framework/src/suricata/src/app-layer-ssl.h @@ -81,6 +81,8 @@ enum { /* flags for file storage */ #define SSL_AL_FLAG_STATE_STORED 0x40000 +#define SSL_AL_FLAG_STATE_LOGGED_LUA 0x80000 + /* config flags */ #define SSL_TLS_LOG_PEM (1 << 0) diff --git a/framework/src/suricata/src/app-layer-template.c b/framework/src/suricata/src/app-layer-template.c index 87b64b73..5eafa67e 100644 --- a/framework/src/suricata/src/app-layer-template.c +++ b/framework/src/suricata/src/app-layer-template.c @@ -25,6 +25,7 @@ #include "suricata-common.h" #include "stream.h" +#include "conf.h" #include "util-unittest.h" @@ -428,6 +429,12 @@ void RegisterTemplateParsers(void) { char *proto_name = "template"; + /* TEMPLATE_START_REMOVE */ + if (ConfGetNode("app-layer.protocols.template") == NULL) { + return; + } + /* TEMPLATE_END_REMOVE */ + /* Check if Template TCP detection is enabled. If it does not exist in * the configuration file then it will be enabled by default. */ if (AppLayerProtoDetectConfProtoDetectionEnabled("tcp", proto_name)) { @@ -481,8 +488,8 @@ void RegisterTemplateParsers(void) STREAM_TOSERVER, TemplateParseRequest); /* Register response parser for parsing frames from server to client. */ - AppLayerParserRegisterParser(IPPROTO_TCP, ALPROTO_TEMPLATE, STREAM_TOCLIENT, - TemplateParseResponse); + AppLayerParserRegisterParser(IPPROTO_TCP, ALPROTO_TEMPLATE, + STREAM_TOCLIENT, TemplateParseResponse); /* Register a function to be called by the application layer * when a transaction is to be freed. */ @@ -490,7 +497,8 @@ void RegisterTemplateParsers(void) TemplateStateTxFree); /* Register a function to return the current transaction count. */ - AppLayerParserRegisterGetTxCnt(IPPROTO_TCP, ALPROTO_TEMPLATE, TemplateGetTxCnt); + AppLayerParserRegisterGetTxCnt(IPPROTO_TCP, ALPROTO_TEMPLATE, + TemplateGetTxCnt); /* Transaction handling. */ AppLayerParserRegisterGetStateProgressCompletionStatus(IPPROTO_TCP, diff --git a/framework/src/suricata/src/app-layer.c b/framework/src/suricata/src/app-layer.c index c47c2dd6..96fa252c 100644 --- a/framework/src/suricata/src/app-layer.c +++ b/framework/src/suricata/src/app-layer.c @@ -74,6 +74,12 @@ static void DisableAppLayer(Flow *f) StreamTcpDisableAppLayer(f); } +static inline int ProtoDetectDone(const Flow *f, const TcpSession *ssn, uint8_t direction) { + const TcpStream *stream = (direction & STREAM_TOSERVER) ? &ssn->client : &ssn->server; + return ((stream->flags & STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_COMPLETED) || + (FLOW_IS_PM_DONE(f, direction) && FLOW_IS_PP_DONE(f, direction))); +} + int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, Packet *p, Flow *f, TcpSession *ssn, TcpStream *stream, @@ -187,7 +193,14 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, p->flowflags |= FLOW_PKT_TOCLIENT; } } - int ret = StreamTcpReassembleAppLayer(tv, ra_ctx, ssn, + + int ret = 0; + /* if the opposing side is not going to work, then + * we just have to give up. */ + if (opposing_stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY) + ret = -1; + else + ret = StreamTcpReassembleAppLayer(tv, ra_ctx, ssn, opposing_stream, p); if (stream == &ssn->client) { if (StreamTcpInlineMode()) { @@ -329,7 +342,7 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, } else { f->data_al_so_far[dir] = data_len; } - } else { + } else { /* See if we're going to have to give up: * * If we're getting a lot of data in one direction and the @@ -349,9 +362,16 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, uint32_t size_ts = ssn->client.last_ack - ssn->client.isn - 1; uint32_t size_tc = ssn->server.last_ack - ssn->server.isn - 1; SCLogDebug("size_ts %u, size_tc %u", size_ts, size_tc); - - if (FLOW_IS_PM_DONE(f, STREAM_TOSERVER) && FLOW_IS_PP_DONE(f, STREAM_TOSERVER) && - FLOW_IS_PM_DONE(f, STREAM_TOCLIENT) && FLOW_IS_PP_DONE(f, STREAM_TOCLIENT)) { +#ifdef DEBUG_VALIDATION + if (!(ssn->client.flags & STREAMTCP_STREAM_FLAG_GAP)) + BUG_ON(size_ts > 1000000UL); + if (!(ssn->server.flags & STREAMTCP_STREAM_FLAG_GAP)) + BUG_ON(size_tc > 1000000UL); +#endif /* DEBUG_VALIDATION */ + + if (ProtoDetectDone(f, ssn, STREAM_TOSERVER) && + ProtoDetectDone(f, ssn, STREAM_TOCLIENT)) + { DisableAppLayer(f); ssn->data_first_seen_dir = APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER; @@ -391,8 +411,19 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, ssn->data_first_seen_dir = APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER; AppLayerDecoderEventsSetEventRaw(&p->app_layer_events, APPLAYER_PROTO_DETECTION_SKIPPED); + /* in case of really low TS data (e.g. 4 bytes) we can have + * the PP complete, PM not complete (depth not reached) and + * the TC side also not recognized (proto unknown) */ + } else if (size_tc > 100000 && + FLOW_IS_PP_DONE(f, STREAM_TOSERVER) && !(FLOW_IS_PM_DONE(f, STREAM_TOSERVER)) && + (!FLOW_IS_PM_DONE(f, STREAM_TOCLIENT) && !FLOW_IS_PP_DONE(f, STREAM_TOCLIENT))) + { + DisableAppLayer(f); + ssn->data_first_seen_dir = APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER; + AppLayerDecoderEventsSetEventRaw(&p->app_layer_events, + APPLAYER_PROTO_DETECTION_SKIPPED); } - } + } } } else { SCLogDebug("stream data (len %" PRIu32 " alproto " diff --git a/framework/src/suricata/src/decode.c b/framework/src/suricata/src/decode.c index 0dd9fa86..4be4b9e7 100644 --- a/framework/src/suricata/src/decode.c +++ b/framework/src/suricata/src/decode.c @@ -403,6 +403,7 @@ void DecodeRegisterPerfCounters(DecodeThreadVars *dtv, ThreadVars *tv) dtv->counter_avg_pkt_size = StatsRegisterAvgCounter("decoder.avg_pkt_size", tv); dtv->counter_max_pkt_size = StatsRegisterMaxCounter("decoder.max_pkt_size", tv); dtv->counter_erspan = StatsRegisterMaxCounter("decoder.erspan", tv); + dtv->counter_flow_memcap = StatsRegisterCounter("flow.memcap", tv); dtv->counter_defrag_ipv4_fragments = StatsRegisterCounter("defrag.ipv4.fragments", tv); diff --git a/framework/src/suricata/src/decode.h b/framework/src/suricata/src/decode.h index 2f322a02..f57dcea9 100644 --- a/framework/src/suricata/src/decode.h +++ b/framework/src/suricata/src/decode.h @@ -624,6 +624,8 @@ typedef struct DecodeThreadVars_ uint16_t counter_defrag_ipv6_timeouts; uint16_t counter_defrag_max_hit; + uint16_t counter_flow_memcap; + /* thread data for flow logging api: only used at forced * flow recycle during lookups */ void *output_flow_thread_data; diff --git a/framework/src/suricata/src/defrag-hash.c b/framework/src/suricata/src/defrag-hash.c index 9cb377e5..e37a0873 100644 --- a/framework/src/suricata/src/defrag-hash.c +++ b/framework/src/suricata/src/defrag-hash.c @@ -96,24 +96,13 @@ static void DefragTrackerInit(DefragTracker *dt, Packet *p) dt->vlan_id[1] = p->vlan_id[1]; dt->policy = DefragGetOsPolicy(p); dt->host_timeout = DefragPolicyGetHostTimeout(p); + dt->remove = 0; + dt->seen_last = 0; TAILQ_INIT(&dt->frags); (void) DefragTrackerIncrUsecnt(dt); } -static DefragTracker *DefragTrackerNew(Packet *p) -{ - DefragTracker *dt = DefragTrackerAlloc(); - if (dt == NULL) - goto error; - - DefragTrackerInit(dt, p); - return dt; - -error: - return NULL; -} - void DefragTrackerRelease(DefragTracker *t) { (void) DefragTrackerDecrUsecnt(t); @@ -467,7 +456,7 @@ static DefragTracker *DefragTrackerGetNew(Packet *p) /* freed a tracker, but it's unlocked */ } else { /* now see if we can alloc a new tracker */ - dt = DefragTrackerNew(p); + dt = DefragTrackerAlloc(); if (dt == NULL) { return NULL; } @@ -526,7 +515,7 @@ DefragTracker *DefragGetTrackerFromHash (Packet *p) dt = hb->head; /* see if this is the tracker we are looking for */ - if (DefragTrackerCompare(dt, p) == 0) { + if (dt->remove || DefragTrackerCompare(dt, p) == 0) { DefragTracker *pdt = NULL; /* previous tracker */ while (dt) { diff --git a/framework/src/suricata/src/defrag.c b/framework/src/suricata/src/defrag.c index 7418b93c..1d86c448 100644 --- a/framework/src/suricata/src/defrag.c +++ b/framework/src/suricata/src/defrag.c @@ -337,6 +337,10 @@ Defrag4Reassemble(ThreadVars *tv, DefragTracker *tracker, Packet *p) if (frag->offset + frag->data_len > fragmentable_len) fragmentable_len = frag->offset + frag->data_len; } + + if (!frag->more_frags) { + break; + } } SCLogDebug("ip_hdr_offset %u, hlen %u, fragmentable_len %u", @@ -460,6 +464,10 @@ Defrag6Reassemble(ThreadVars *tv, DefragTracker *tracker, Packet *p) if (frag->offset + frag->data_len > fragmentable_len) fragmentable_len = frag->offset + frag->data_len; } + + if (!frag->more_frags) { + break; + } } rp->ip6h = (IPV6Hdr *)(GET_PKT_DATA(rp) + ip_hdr_offset); @@ -747,6 +755,7 @@ insert: new->data_len = data_len - ltrim; new->ip_hdr_offset = ip_hdr_offset; new->frag_hdr_offset = frag_hdr_offset; + new->more_frags = more_frags; #ifdef DEBUG new->pcap_cnt = pcap_cnt; #endif @@ -2328,6 +2337,198 @@ end: return ret; } +static int DefragTrackerReuseTest(void) +{ + int ret = 0; + int id = 1; + Packet *p1 = NULL; + DefragTracker *tracker1 = NULL, *tracker2 = NULL; + + DefragInit(); + + /* Build a packet, its not a fragment but shouldn't matter for + * this test. */ + p1 = BuildTestPacket(id, 0, 0, 'A', 8); + if (p1 == NULL) { + goto end; + } + + /* Get a tracker. It shouldn't look like its already in use. */ + tracker1 = DefragGetTracker(NULL, NULL, p1); + if (tracker1 == NULL) { + goto end; + } + if (tracker1->seen_last) { + goto end; + } + if (tracker1->remove) { + goto end; + } + DefragTrackerRelease(tracker1); + + /* Get a tracker again, it should be the same one. */ + tracker2 = DefragGetTracker(NULL, NULL, p1); + if (tracker2 == NULL) { + goto end; + } + if (tracker2 != tracker1) { + goto end; + } + DefragTrackerRelease(tracker1); + + /* Now mark the tracker for removal. It should not be returned + * when we get a tracker for a packet that may have the same + * attributes. */ + tracker1->remove = 1; + + tracker2 = DefragGetTracker(NULL, NULL, p1); + if (tracker2 == NULL) { + goto end; + } + if (tracker2 == tracker1) { + goto end; + } + if (tracker2->remove) { + goto end; + } + + ret = 1; +end: + if (p1 != NULL) { + SCFree(p1); + } + DefragDestroy(); + return ret; +} + +/** + * IPV4: Test the case where you have a packet fragmented in 3 parts + * and send like: + * - Offset: 2; MF: 1 + * - Offset: 0; MF: 1 + * - Offset: 1; MF: 0 + * + * Only the fragments with offset 0 and 1 should be reassembled. + */ +static int DefragMfIpv4Test(void) +{ + int retval = 0; + int ip_id = 9; + Packet *p = NULL; + + DefragInit(); + + Packet *p1 = BuildTestPacket(ip_id, 2, 1, 'C', 8); + Packet *p2 = BuildTestPacket(ip_id, 0, 1, 'A', 8); + Packet *p3 = BuildTestPacket(ip_id, 1, 0, 'B', 8); + if (p1 == NULL || p2 == NULL || p3 == NULL) { + goto end; + } + + p = Defrag(NULL, NULL, p1, NULL); + if (p != NULL) { + goto end; + } + + p = Defrag(NULL, NULL, p2, NULL); + if (p != NULL) { + goto end; + } + + /* This should return a packet as MF=0. */ + p = Defrag(NULL, NULL, p3, NULL); + if (p == NULL) { + goto end; + } + + /* Expected IP length is 20 + 8 + 8 = 36 as only 2 of the + * fragments should be in the re-assembled packet. */ + if (IPV4_GET_IPLEN(p) != 36) { + goto end; + } + + retval = 1; +end: + if (p1 != NULL) { + SCFree(p1); + } + if (p2 != NULL) { + SCFree(p2); + } + if (p3 != NULL) { + SCFree(p3); + } + if (p != NULL) { + SCFree(p); + } + DefragDestroy(); + return retval; +} + +/** + * IPV6: Test the case where you have a packet fragmented in 3 parts + * and send like: + * - Offset: 2; MF: 1 + * - Offset: 0; MF: 1 + * - Offset: 1; MF: 0 + * + * Only the fragments with offset 0 and 1 should be reassembled. + */ +static int DefragMfIpv6Test(void) +{ + int retval = 0; + int ip_id = 9; + Packet *p = NULL; + + DefragInit(); + + Packet *p1 = IPV6BuildTestPacket(ip_id, 2, 1, 'C', 8); + Packet *p2 = IPV6BuildTestPacket(ip_id, 0, 1, 'A', 8); + Packet *p3 = IPV6BuildTestPacket(ip_id, 1, 0, 'B', 8); + if (p1 == NULL || p2 == NULL || p3 == NULL) { + goto end; + } + + p = Defrag(NULL, NULL, p1, NULL); + if (p != NULL) { + goto end; + } + + p = Defrag(NULL, NULL, p2, NULL); + if (p != NULL) { + goto end; + } + + /* This should return a packet as MF=0. */ + p = Defrag(NULL, NULL, p3, NULL); + if (p == NULL) { + goto end; + } + + /* For IPv6 the expected length is just the length of the payload + * of 2 fragments, so 16. */ + if (IPV6_GET_PLEN(p) != 16) { + goto end; + } + + retval = 1; +end: + if (p1 != NULL) { + SCFree(p1); + } + if (p2 != NULL) { + SCFree(p2); + } + if (p3 != NULL) { + SCFree(p3); + } + if (p != NULL) { + SCFree(p); + } + DefragDestroy(); + return retval; +} + #endif /* UNITTESTS */ void @@ -2373,9 +2574,11 @@ DefragRegisterTests(void) UtRegisterTest("DefragVlanTest", DefragVlanTest, 1); UtRegisterTest("DefragVlanQinQTest", DefragVlanQinQTest, 1); - + UtRegisterTest("DefragTrackerReuseTest", DefragTrackerReuseTest, 1); UtRegisterTest("DefragTimeoutTest", DefragTimeoutTest, 1); + UtRegisterTest("DefragMfIpv4Test", DefragMfIpv4Test, 1); + UtRegisterTest("DefragMfIpv6Test", DefragMfIpv6Test, 1); #endif /* UNITTESTS */ } diff --git a/framework/src/suricata/src/defrag.h b/framework/src/suricata/src/defrag.h index 8bd0325a..2dfaec60 100644 --- a/framework/src/suricata/src/defrag.h +++ b/framework/src/suricata/src/defrag.h @@ -71,20 +71,6 @@ typedef struct Frag_ { TAILQ_ENTRY(Frag_) next; /**< Pointer to next fragment for tailq. */ } Frag; -/** \brief Reset tracker fields except "lock" */ -#define DEFRAG_TRACKER_RESET(t) { \ - (t)->timeout = 0; \ - (t)->id = 0; \ - (t)->policy = 0; \ - (t)->af = 0; \ - (t)->seen_last = 0; \ - (t)->remove = 0; \ - CLEAR_ADDR(&(t)->src_addr); \ - CLEAR_ADDR(&(t)->dst_addr); \ - (t)->frags.tqh_first = NULL; \ - (t)->frags.tqh_last = NULL; \ -} - /** * A defragmentation tracker. Used to track fragments that make up a * single packet. diff --git a/framework/src/suricata/src/detect-engine-filedata-smtp.c b/framework/src/suricata/src/detect-engine-filedata-smtp.c index dc50d8c7..829832f9 100644 --- a/framework/src/suricata/src/detect-engine-filedata-smtp.c +++ b/framework/src/suricata/src/detect-engine-filedata-smtp.c @@ -267,7 +267,7 @@ int DetectEngineRunSMTPMpm(DetectEngineCtx *de_ctx, uint32_t cnt = 0; uint32_t buffer_len = 0; uint32_t stream_start_offset = 0; - uint8_t *buffer = 0; + uint8_t *buffer = NULL; if (ffc != NULL) { File *file = ffc->head; @@ -278,11 +278,10 @@ int DetectEngineRunSMTPMpm(DetectEngineCtx *de_ctx, flags, &buffer_len, &stream_start_offset); - if (buffer_len == 0) goto end; - cnt = SMTPFiledataPatternSearch(det_ctx, buffer, buffer_len, flags); + cnt += SMTPFiledataPatternSearch(det_ctx, buffer, buffer_len, flags); } } end: diff --git a/framework/src/suricata/src/detect-engine-mpm.c b/framework/src/suricata/src/detect-engine-mpm.c index c34ef252..7fd532e5 100644 --- a/framework/src/suricata/src/detect-engine-mpm.c +++ b/framework/src/suricata/src/detect-engine-mpm.c @@ -58,6 +58,7 @@ #ifdef __SC_CUDA_SUPPORT__ #include "util-mpm-ac.h" #endif +#include "util-validate.h" /** \todo make it possible to use multiple pattern matcher algorithms next to each other. */ @@ -83,7 +84,7 @@ int SignatureHasPacketContent(Signature *s) SCReturnInt(0); } - if (!(s->proto.proto[6 / 8] & 1 << (6 % 8))) { + if (!(s->proto.proto[IPPROTO_TCP / 8] & 1 << (IPPROTO_TCP % 8))) { SCReturnInt(1); } @@ -117,7 +118,7 @@ int SignatureHasStreamContent(Signature *s) SCReturnInt(0); } - if (!(s->proto.proto[6 / 8] & 1 << (6 % 8))) { + if (!(s->proto.proto[IPPROTO_TCP / 8] & 1 << (IPPROTO_TCP % 8))) { SCReturnInt(0); } @@ -170,7 +171,7 @@ uint16_t PatternMatchDefaultMatcher(void) done: #ifdef __tile__ if (mpm_algo_val == MPM_AC) - mpm_algo_val = MPM_AC_TILE; + mpm_algo_val = MPM_AC_TILE; #endif return mpm_algo_val; @@ -184,15 +185,13 @@ uint32_t PacketPatternSearchWithStreamCtx(DetectEngineThreadCtx *det_ctx, uint32_t ret = 0; if (p->flowflags & FLOW_PKT_TOSERVER) { - if (det_ctx->sgh->mpm_stream_ctx_ts == NULL) - SCReturnInt(0); + DEBUG_VALIDATE_BUG_ON(det_ctx->sgh->mpm_stream_ctx_ts == NULL); ret = mpm_table[det_ctx->sgh->mpm_stream_ctx_ts->mpm_type]. Search(det_ctx->sgh->mpm_stream_ctx_ts, &det_ctx->mtc, &det_ctx->pmq, p->payload, p->payload_len); } else { - if (det_ctx->sgh->mpm_stream_ctx_tc == NULL) - SCReturnInt(0); + DEBUG_VALIDATE_BUG_ON(det_ctx->sgh->mpm_stream_ctx_tc == NULL); ret = mpm_table[det_ctx->sgh->mpm_stream_ctx_tc->mpm_type]. Search(det_ctx->sgh->mpm_stream_ctx_tc, &det_ctx->mtc, &det_ctx->pmq, @@ -231,8 +230,7 @@ uint32_t PacketPatternSearch(DetectEngineThreadCtx *det_ctx, Packet *p) } else { mpm_ctx = det_ctx->sgh->mpm_proto_other_ctx; } - - if (mpm_ctx == NULL) + if (unlikely(mpm_ctx == NULL)) SCReturnInt(0); #ifdef __SC_CUDA_SUPPORT__ @@ -269,16 +267,13 @@ uint32_t UriPatternSearch(DetectEngineThreadCtx *det_ctx, SCEnter(); uint32_t ret; - if (flags & STREAM_TOSERVER) { - if (det_ctx->sgh->mpm_uri_ctx_ts == NULL) - SCReturnUInt(0U); - ret = mpm_table[det_ctx->sgh->mpm_uri_ctx_ts->mpm_type]. - Search(det_ctx->sgh->mpm_uri_ctx_ts, - &det_ctx->mtcu, &det_ctx->pmq, uri, uri_len); - } else { - BUG_ON(1); - } + DEBUG_VALIDATE_BUG_ON(flags & STREAM_TOCLIENT); + DEBUG_VALIDATE_BUG_ON(det_ctx->sgh->mpm_uri_ctx_ts == NULL); + + ret = mpm_table[det_ctx->sgh->mpm_uri_ctx_ts->mpm_type]. + Search(det_ctx->sgh->mpm_uri_ctx_ts, + &det_ctx->mtcu, &det_ctx->pmq, uri, uri_len); //PrintRawDataFp(stdout, uri, uri_len); @@ -300,16 +295,13 @@ uint32_t HttpClientBodyPatternSearch(DetectEngineThreadCtx *det_ctx, SCEnter(); uint32_t ret; - if (flags & STREAM_TOSERVER) { - if (det_ctx->sgh->mpm_hcbd_ctx_ts == NULL) - SCReturnUInt(0); - ret = mpm_table[det_ctx->sgh->mpm_hcbd_ctx_ts->mpm_type]. - Search(det_ctx->sgh->mpm_hcbd_ctx_ts, &det_ctx->mtcu, - &det_ctx->pmq, body, body_len); - } else { - BUG_ON(1); - } + DEBUG_VALIDATE_BUG_ON(flags & STREAM_TOCLIENT); + DEBUG_VALIDATE_BUG_ON(det_ctx->sgh->mpm_hcbd_ctx_ts == NULL); + + ret = mpm_table[det_ctx->sgh->mpm_hcbd_ctx_ts->mpm_type]. + Search(det_ctx->sgh->mpm_hcbd_ctx_ts, &det_ctx->mtcu, + &det_ctx->pmq, body, body_len); SCReturnUInt(ret); } @@ -329,16 +321,13 @@ uint32_t HttpServerBodyPatternSearch(DetectEngineThreadCtx *det_ctx, SCEnter(); uint32_t ret; - if (flags & STREAM_TOSERVER) { - BUG_ON(1); - } else { - if (det_ctx->sgh->mpm_hsbd_ctx_tc == NULL) - SCReturnUInt(0); - ret = mpm_table[det_ctx->sgh->mpm_hsbd_ctx_tc->mpm_type]. - Search(det_ctx->sgh->mpm_hsbd_ctx_tc, &det_ctx->mtcu, - &det_ctx->pmq, body, body_len); - } + DEBUG_VALIDATE_BUG_ON(!(flags & STREAM_TOCLIENT)); + DEBUG_VALIDATE_BUG_ON(det_ctx->sgh->mpm_hsbd_ctx_tc == NULL); + + ret = mpm_table[det_ctx->sgh->mpm_hsbd_ctx_tc->mpm_type]. + Search(det_ctx->sgh->mpm_hsbd_ctx_tc, &det_ctx->mtcu, + &det_ctx->pmq, body, body_len); SCReturnUInt(ret); } @@ -359,15 +348,13 @@ uint32_t HttpHeaderPatternSearch(DetectEngineThreadCtx *det_ctx, uint32_t ret; if (flags & STREAM_TOSERVER) { - if (det_ctx->sgh->mpm_hhd_ctx_ts == NULL) - SCReturnUInt(0); + DEBUG_VALIDATE_BUG_ON(det_ctx->sgh->mpm_hhd_ctx_ts == NULL); ret = mpm_table[det_ctx->sgh->mpm_hhd_ctx_ts->mpm_type]. Search(det_ctx->sgh->mpm_hhd_ctx_ts, &det_ctx->mtcu, &det_ctx->pmq, headers, headers_len); } else { - if (det_ctx->sgh->mpm_hhd_ctx_tc == NULL) - SCReturnUInt(0); + DEBUG_VALIDATE_BUG_ON(det_ctx->sgh->mpm_hhd_ctx_tc == NULL); ret = mpm_table[det_ctx->sgh->mpm_hhd_ctx_tc->mpm_type]. Search(det_ctx->sgh->mpm_hhd_ctx_tc, &det_ctx->mtcu, @@ -393,15 +380,13 @@ uint32_t HttpRawHeaderPatternSearch(DetectEngineThreadCtx *det_ctx, uint32_t ret; if (flags & STREAM_TOSERVER) { - if (det_ctx->sgh->mpm_hrhd_ctx_ts == NULL) - SCReturnUInt(0); + DEBUG_VALIDATE_BUG_ON(det_ctx->sgh->mpm_hrhd_ctx_ts == NULL); ret = mpm_table[det_ctx->sgh->mpm_hrhd_ctx_ts->mpm_type]. Search(det_ctx->sgh->mpm_hrhd_ctx_ts, &det_ctx->mtcu, &det_ctx->pmq, raw_headers, raw_headers_len); } else { - if (det_ctx->sgh->mpm_hrhd_ctx_tc == NULL) - SCReturnUInt(0); + DEBUG_VALIDATE_BUG_ON(det_ctx->sgh->mpm_hrhd_ctx_tc == NULL); ret = mpm_table[det_ctx->sgh->mpm_hrhd_ctx_tc->mpm_type]. Search(det_ctx->sgh->mpm_hrhd_ctx_tc, &det_ctx->mtcu, @@ -426,16 +411,13 @@ uint32_t HttpMethodPatternSearch(DetectEngineThreadCtx *det_ctx, SCEnter(); uint32_t ret; - if (flags & STREAM_TOSERVER) { - if (det_ctx->sgh->mpm_hmd_ctx_ts == NULL) - SCReturnUInt(0); - ret = mpm_table[det_ctx->sgh->mpm_hmd_ctx_ts->mpm_type]. - Search(det_ctx->sgh->mpm_hmd_ctx_ts, &det_ctx->mtcu, - &det_ctx->pmq, raw_method, raw_method_len); - } else { - BUG_ON(1); - } + DEBUG_VALIDATE_BUG_ON(flags & STREAM_TOCLIENT); + DEBUG_VALIDATE_BUG_ON(det_ctx->sgh->mpm_hmd_ctx_ts == NULL); + + ret = mpm_table[det_ctx->sgh->mpm_hmd_ctx_ts->mpm_type]. + Search(det_ctx->sgh->mpm_hmd_ctx_ts, &det_ctx->mtcu, + &det_ctx->pmq, raw_method, raw_method_len); SCReturnUInt(ret); } @@ -456,15 +438,13 @@ uint32_t HttpCookiePatternSearch(DetectEngineThreadCtx *det_ctx, uint32_t ret; if (flags & STREAM_TOSERVER) { - if (det_ctx->sgh->mpm_hcd_ctx_ts == NULL) - SCReturnUInt(0); + DEBUG_VALIDATE_BUG_ON(det_ctx->sgh->mpm_hcd_ctx_ts == NULL); ret = mpm_table[det_ctx->sgh->mpm_hcd_ctx_ts->mpm_type]. Search(det_ctx->sgh->mpm_hcd_ctx_ts, &det_ctx->mtcu, &det_ctx->pmq, cookie, cookie_len); } else { - if (det_ctx->sgh->mpm_hcd_ctx_tc == NULL) - SCReturnUInt(0); + DEBUG_VALIDATE_BUG_ON(det_ctx->sgh->mpm_hcd_ctx_tc == NULL); ret = mpm_table[det_ctx->sgh->mpm_hcd_ctx_tc->mpm_type]. Search(det_ctx->sgh->mpm_hcd_ctx_tc, &det_ctx->mtcu, @@ -489,16 +469,13 @@ uint32_t HttpRawUriPatternSearch(DetectEngineThreadCtx *det_ctx, SCEnter(); uint32_t ret; - if (flags & STREAM_TOSERVER) { - if (det_ctx->sgh->mpm_hrud_ctx_ts == NULL) - SCReturnUInt(0); - ret = mpm_table[det_ctx->sgh->mpm_hrud_ctx_ts->mpm_type]. - Search(det_ctx->sgh->mpm_hrud_ctx_ts, &det_ctx->mtcu, - &det_ctx->pmq, uri, uri_len); - } else { - BUG_ON(1); - } + DEBUG_VALIDATE_BUG_ON(flags & STREAM_TOCLIENT); + DEBUG_VALIDATE_BUG_ON(det_ctx->sgh->mpm_hrud_ctx_ts == NULL); + + ret = mpm_table[det_ctx->sgh->mpm_hrud_ctx_ts->mpm_type]. + Search(det_ctx->sgh->mpm_hrud_ctx_ts, &det_ctx->mtcu, + &det_ctx->pmq, uri, uri_len); SCReturnUInt(ret); } @@ -518,16 +495,13 @@ uint32_t HttpStatMsgPatternSearch(DetectEngineThreadCtx *det_ctx, SCEnter(); uint32_t ret; - if (flags & STREAM_TOSERVER) { - BUG_ON(1); - } else { - if (det_ctx->sgh->mpm_hsmd_ctx_tc == NULL) - SCReturnUInt(0); - ret = mpm_table[det_ctx->sgh->mpm_hsmd_ctx_tc->mpm_type]. - Search(det_ctx->sgh->mpm_hsmd_ctx_tc, &det_ctx->mtcu, - &det_ctx->pmq, stat_msg, stat_msg_len); - } + DEBUG_VALIDATE_BUG_ON(!(flags & STREAM_TOCLIENT)); + DEBUG_VALIDATE_BUG_ON(det_ctx->sgh->mpm_hsmd_ctx_tc == NULL); + + ret = mpm_table[det_ctx->sgh->mpm_hsmd_ctx_tc->mpm_type]. + Search(det_ctx->sgh->mpm_hsmd_ctx_tc, &det_ctx->mtcu, + &det_ctx->pmq, stat_msg, stat_msg_len); SCReturnUInt(ret); } @@ -547,16 +521,13 @@ uint32_t HttpStatCodePatternSearch(DetectEngineThreadCtx *det_ctx, SCEnter(); uint32_t ret; - if (flags & STREAM_TOSERVER) { - BUG_ON(1); - } else { - if (det_ctx->sgh->mpm_hscd_ctx_tc == NULL) - SCReturnUInt(0); - ret = mpm_table[det_ctx->sgh->mpm_hscd_ctx_tc->mpm_type]. - Search(det_ctx->sgh->mpm_hscd_ctx_tc, &det_ctx->mtcu, - &det_ctx->pmq, stat_code, stat_code_len); - } + DEBUG_VALIDATE_BUG_ON(!(flags & STREAM_TOCLIENT)); + DEBUG_VALIDATE_BUG_ON(det_ctx->sgh->mpm_hscd_ctx_tc == NULL); + + ret = mpm_table[det_ctx->sgh->mpm_hscd_ctx_tc->mpm_type]. + Search(det_ctx->sgh->mpm_hscd_ctx_tc, &det_ctx->mtcu, + &det_ctx->pmq, stat_code, stat_code_len); SCReturnUInt(ret); } @@ -576,16 +547,13 @@ uint32_t HttpUAPatternSearch(DetectEngineThreadCtx *det_ctx, SCEnter(); uint32_t ret; - if (flags & STREAM_TOSERVER) { - if (det_ctx->sgh->mpm_huad_ctx_ts == NULL) - SCReturnUInt(0); - ret = mpm_table[det_ctx->sgh->mpm_huad_ctx_ts->mpm_type]. - Search(det_ctx->sgh->mpm_huad_ctx_ts, &det_ctx->mtcu, - &det_ctx->pmq, ua, ua_len); - } else { - BUG_ON(1); - } + DEBUG_VALIDATE_BUG_ON(flags & STREAM_TOCLIENT); + DEBUG_VALIDATE_BUG_ON(det_ctx->sgh->mpm_huad_ctx_ts == NULL); + + ret = mpm_table[det_ctx->sgh->mpm_huad_ctx_ts->mpm_type]. + Search(det_ctx->sgh->mpm_huad_ctx_ts, &det_ctx->mtcu, + &det_ctx->pmq, ua, ua_len); SCReturnUInt(ret); } @@ -606,16 +574,13 @@ uint32_t HttpHHPatternSearch(DetectEngineThreadCtx *det_ctx, SCEnter(); uint32_t ret; - if (flags & STREAM_TOSERVER) { - if (det_ctx->sgh->mpm_hhhd_ctx_ts == NULL) - SCReturnUInt(0); - ret = mpm_table[det_ctx->sgh->mpm_hhhd_ctx_ts->mpm_type]. - Search(det_ctx->sgh->mpm_hhhd_ctx_ts, &det_ctx->mtcu, - &det_ctx->pmq, hh, hh_len); - } else { - BUG_ON(1); - } + DEBUG_VALIDATE_BUG_ON(flags & STREAM_TOCLIENT); + DEBUG_VALIDATE_BUG_ON(det_ctx->sgh->mpm_hhhd_ctx_ts == NULL); + + ret = mpm_table[det_ctx->sgh->mpm_hhhd_ctx_ts->mpm_type]. + Search(det_ctx->sgh->mpm_hhhd_ctx_ts, &det_ctx->mtcu, + &det_ctx->pmq, hh, hh_len); SCReturnUInt(ret); } @@ -636,16 +601,13 @@ uint32_t HttpHRHPatternSearch(DetectEngineThreadCtx *det_ctx, SCEnter(); uint32_t ret; - if (flags & STREAM_TOSERVER) { - if (det_ctx->sgh->mpm_hrhhd_ctx_ts == NULL) - SCReturnUInt(0); - ret = mpm_table[det_ctx->sgh->mpm_hrhhd_ctx_ts->mpm_type]. - Search(det_ctx->sgh->mpm_hrhhd_ctx_ts, &det_ctx->mtcu, - &det_ctx->pmq, hrh, hrh_len); - } else { - BUG_ON(1); - } + DEBUG_VALIDATE_BUG_ON(flags & STREAM_TOCLIENT); + DEBUG_VALIDATE_BUG_ON(det_ctx->sgh->mpm_hrhhd_ctx_ts == NULL); + + ret = mpm_table[det_ctx->sgh->mpm_hrhhd_ctx_ts->mpm_type]. + Search(det_ctx->sgh->mpm_hrhhd_ctx_ts, &det_ctx->mtcu, + &det_ctx->pmq, hrh, hrh_len); SCReturnUInt(ret); } @@ -668,14 +630,12 @@ uint32_t DnsQueryPatternSearch(DetectEngineThreadCtx *det_ctx, uint32_t ret = 0; - if (flags & STREAM_TOSERVER) { - if (det_ctx->sgh->mpm_dnsquery_ctx_ts == NULL) - SCReturnUInt(0); + DEBUG_VALIDATE_BUG_ON(flags & STREAM_TOCLIENT); + DEBUG_VALIDATE_BUG_ON(det_ctx->sgh->mpm_dnsquery_ctx_ts == NULL); - ret = mpm_table[det_ctx->sgh->mpm_dnsquery_ctx_ts->mpm_type]. - Search(det_ctx->sgh->mpm_dnsquery_ctx_ts, &det_ctx->mtcu, - &det_ctx->pmq, buffer, buffer_len); - } + ret = mpm_table[det_ctx->sgh->mpm_dnsquery_ctx_ts->mpm_type]. + Search(det_ctx->sgh->mpm_dnsquery_ctx_ts, &det_ctx->mtcu, + &det_ctx->pmq, buffer, buffer_len); SCReturnUInt(ret); } @@ -755,14 +715,12 @@ uint32_t SMTPFiledataPatternSearch(DetectEngineThreadCtx *det_ctx, uint32_t ret = 0; - if (flags & STREAM_TOSERVER) { - if (det_ctx->sgh->mpm_smtp_filedata_ctx_ts == NULL) - SCReturnUInt(0); + DEBUG_VALIDATE_BUG_ON(flags & STREAM_TOCLIENT); + DEBUG_VALIDATE_BUG_ON(det_ctx->sgh->mpm_smtp_filedata_ctx_ts == NULL); - ret = mpm_table[det_ctx->sgh->mpm_smtp_filedata_ctx_ts->mpm_type]. - Search(det_ctx->sgh->mpm_smtp_filedata_ctx_ts, &det_ctx->mtcu, - &det_ctx->pmq, buffer, buffer_len); - } + ret = mpm_table[det_ctx->sgh->mpm_smtp_filedata_ctx_ts->mpm_type]. + Search(det_ctx->sgh->mpm_smtp_filedata_ctx_ts, &det_ctx->mtcu, + &det_ctx->pmq, buffer, buffer_len); SCReturnUInt(ret); } @@ -862,10 +820,11 @@ void PatternMatchDestroyGroup(SigGroupHead *sh) { /* content */ if (!(sh->flags & SIG_GROUP_HEAD_MPM_COPY)) { - SCLogDebug("destroying mpm_ctx %p (sh %p)", - sh->mpm_proto_tcp_ctx_ts, sh); if (sh->mpm_proto_tcp_ctx_ts != NULL && - !sh->mpm_proto_tcp_ctx_ts->global) { + !sh->mpm_proto_tcp_ctx_ts->global) + { + SCLogDebug("destroying mpm_ctx %p (sh %p)", + sh->mpm_proto_tcp_ctx_ts, sh); mpm_table[sh->mpm_proto_tcp_ctx_ts->mpm_type]. DestroyCtx(sh->mpm_proto_tcp_ctx_ts); SCFree(sh->mpm_proto_tcp_ctx_ts); @@ -873,10 +832,11 @@ void PatternMatchDestroyGroup(SigGroupHead *sh) /* ready for reuse */ sh->mpm_proto_tcp_ctx_ts = NULL; - SCLogDebug("destroying mpm_ctx %p (sh %p)", - sh->mpm_proto_tcp_ctx_tc, sh); if (sh->mpm_proto_tcp_ctx_tc != NULL && - !sh->mpm_proto_tcp_ctx_tc->global) { + !sh->mpm_proto_tcp_ctx_tc->global) + { + SCLogDebug("destroying mpm_ctx %p (sh %p)", + sh->mpm_proto_tcp_ctx_tc, sh); mpm_table[sh->mpm_proto_tcp_ctx_tc->mpm_type]. DestroyCtx(sh->mpm_proto_tcp_ctx_tc); SCFree(sh->mpm_proto_tcp_ctx_tc); @@ -884,10 +844,11 @@ void PatternMatchDestroyGroup(SigGroupHead *sh) /* ready for reuse */ sh->mpm_proto_tcp_ctx_tc = NULL; - SCLogDebug("destroying mpm_ctx %p (sh %p)", - sh->mpm_proto_udp_ctx_ts, sh); if (sh->mpm_proto_udp_ctx_ts != NULL && - !sh->mpm_proto_udp_ctx_ts->global) { + !sh->mpm_proto_udp_ctx_ts->global) + { + SCLogDebug("destroying mpm_ctx %p (sh %p)", + sh->mpm_proto_udp_ctx_ts, sh); mpm_table[sh->mpm_proto_udp_ctx_ts->mpm_type]. DestroyCtx(sh->mpm_proto_udp_ctx_ts); SCFree(sh->mpm_proto_udp_ctx_ts); @@ -895,10 +856,11 @@ void PatternMatchDestroyGroup(SigGroupHead *sh) /* ready for reuse */ sh->mpm_proto_udp_ctx_ts = NULL; - SCLogDebug("destroying mpm_ctx %p (sh %p)", - sh->mpm_proto_udp_ctx_tc, sh); if (sh->mpm_proto_udp_ctx_tc != NULL && - !sh->mpm_proto_udp_ctx_tc->global) { + !sh->mpm_proto_udp_ctx_tc->global) + { + SCLogDebug("destroying mpm_ctx %p (sh %p)", + sh->mpm_proto_udp_ctx_tc, sh); mpm_table[sh->mpm_proto_udp_ctx_tc->mpm_type]. DestroyCtx(sh->mpm_proto_udp_ctx_tc); SCFree(sh->mpm_proto_udp_ctx_tc); @@ -906,10 +868,11 @@ void PatternMatchDestroyGroup(SigGroupHead *sh) /* ready for reuse */ sh->mpm_proto_udp_ctx_tc = NULL; - SCLogDebug("destroying mpm_ctx %p (sh %p)", - sh->mpm_proto_other_ctx, sh); if (sh->mpm_proto_other_ctx != NULL && - !sh->mpm_proto_other_ctx->global) { + !sh->mpm_proto_other_ctx->global) + { + SCLogDebug("destroying mpm_ctx %p (sh %p)", + sh->mpm_proto_other_ctx, sh); mpm_table[sh->mpm_proto_other_ctx->mpm_type]. DestroyCtx(sh->mpm_proto_other_ctx); SCFree(sh->mpm_proto_other_ctx); @@ -921,8 +884,8 @@ void PatternMatchDestroyGroup(SigGroupHead *sh) /* uricontent */ if ((sh->mpm_uri_ctx_ts != NULL) && !(sh->flags & SIG_GROUP_HEAD_MPM_URI_COPY)) { if (sh->mpm_uri_ctx_ts != NULL) { - SCLogDebug("destroying mpm_uri_ctx %p (sh %p)", sh->mpm_uri_ctx_ts, sh); if (!sh->mpm_uri_ctx_ts->global) { + SCLogDebug("destroying mpm_uri_ctx %p (sh %p)", sh->mpm_uri_ctx_ts, sh); mpm_table[sh->mpm_uri_ctx_ts->mpm_type].DestroyCtx(sh->mpm_uri_ctx_ts); SCFree(sh->mpm_uri_ctx_ts); } @@ -935,8 +898,8 @@ void PatternMatchDestroyGroup(SigGroupHead *sh) if ((sh->mpm_stream_ctx_ts != NULL || sh->mpm_stream_ctx_tc != NULL) && !(sh->flags & SIG_GROUP_HEAD_MPM_STREAM_COPY)) { if (sh->mpm_stream_ctx_ts != NULL) { - SCLogDebug("destroying mpm_stream_ctx %p (sh %p)", sh->mpm_stream_ctx_ts, sh); if (!sh->mpm_stream_ctx_ts->global) { + SCLogDebug("destroying mpm_stream_ctx %p (sh %p)", sh->mpm_stream_ctx_ts, sh); mpm_table[sh->mpm_stream_ctx_ts->mpm_type].DestroyCtx(sh->mpm_stream_ctx_ts); SCFree(sh->mpm_stream_ctx_ts); } @@ -944,8 +907,8 @@ void PatternMatchDestroyGroup(SigGroupHead *sh) sh->mpm_stream_ctx_ts = NULL; } if (sh->mpm_stream_ctx_tc != NULL) { - SCLogDebug("destroying mpm_stream_ctx %p (sh %p)", sh->mpm_stream_ctx_tc, sh); if (!sh->mpm_stream_ctx_tc->global) { + SCLogDebug("destroying mpm_stream_ctx %p (sh %p)", sh->mpm_stream_ctx_tc, sh); mpm_table[sh->mpm_stream_ctx_tc->mpm_type].DestroyCtx(sh->mpm_stream_ctx_tc); SCFree(sh->mpm_stream_ctx_tc); } @@ -2380,315 +2343,312 @@ int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh) has_co_hrhhd || has_co_dnsquery) { - PatternMatchPreparePopulateMpm(de_ctx, sh); - //if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { - if (sh->mpm_proto_tcp_ctx_ts != NULL) { - if (sh->mpm_proto_tcp_ctx_ts->pattern_cnt == 0) { - MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_proto_tcp_ctx_ts); - sh->mpm_proto_tcp_ctx_ts = NULL; - } else { - if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { - if (mpm_table[sh->mpm_proto_tcp_ctx_ts->mpm_type].Prepare != NULL) { - mpm_table[sh->mpm_proto_tcp_ctx_ts->mpm_type]. - Prepare(sh->mpm_proto_tcp_ctx_ts); - } - } - } - } - if (sh->mpm_proto_tcp_ctx_tc != NULL) { - if (sh->mpm_proto_tcp_ctx_tc->pattern_cnt == 0) { - MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_proto_tcp_ctx_tc); - sh->mpm_proto_tcp_ctx_tc = NULL; - } else { - if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { - if (mpm_table[sh->mpm_proto_tcp_ctx_tc->mpm_type].Prepare != NULL) { - mpm_table[sh->mpm_proto_tcp_ctx_tc->mpm_type]. - Prepare(sh->mpm_proto_tcp_ctx_tc); - } - } - } - } - - if (sh->mpm_proto_udp_ctx_ts != NULL) { - if (sh->mpm_proto_udp_ctx_ts->pattern_cnt == 0) { - MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_proto_udp_ctx_ts); - sh->mpm_proto_udp_ctx_ts = NULL; - } else { - if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { - if (mpm_table[sh->mpm_proto_udp_ctx_ts->mpm_type].Prepare != NULL) { - mpm_table[sh->mpm_proto_udp_ctx_ts->mpm_type]. - Prepare(sh->mpm_proto_udp_ctx_ts); - } - } - } - } - if (sh->mpm_proto_udp_ctx_tc != NULL) { - if (sh->mpm_proto_udp_ctx_tc->pattern_cnt == 0) { - MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_proto_udp_ctx_tc); - sh->mpm_proto_udp_ctx_tc = NULL; - } else { - if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { - if (mpm_table[sh->mpm_proto_udp_ctx_tc->mpm_type].Prepare != NULL) { - mpm_table[sh->mpm_proto_udp_ctx_tc->mpm_type]. - Prepare(sh->mpm_proto_udp_ctx_tc); - } - } - } - } - - if (sh->mpm_proto_other_ctx != NULL) { - if (sh->mpm_proto_other_ctx->pattern_cnt == 0) { - MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_proto_other_ctx); - sh->mpm_proto_other_ctx = NULL; - } else { - if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { - if (mpm_table[sh->mpm_proto_other_ctx->mpm_type].Prepare != NULL) { - mpm_table[sh->mpm_proto_other_ctx->mpm_type]. - Prepare(sh->mpm_proto_other_ctx); - } - } - } - } - - if (sh->mpm_stream_ctx_ts != NULL) { - if (sh->mpm_stream_ctx_ts->pattern_cnt == 0) { - MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_stream_ctx_ts); - sh->mpm_stream_ctx_ts = NULL; - } else { - if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { - if (mpm_table[sh->mpm_stream_ctx_ts->mpm_type].Prepare != NULL) - mpm_table[sh->mpm_stream_ctx_ts->mpm_type].Prepare(sh->mpm_stream_ctx_ts); - } - } - } - if (sh->mpm_stream_ctx_tc != NULL) { - if (sh->mpm_stream_ctx_tc->pattern_cnt == 0) { - MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_stream_ctx_tc); - sh->mpm_stream_ctx_tc = NULL; - } else { - if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { - if (mpm_table[sh->mpm_stream_ctx_tc->mpm_type].Prepare != NULL) - mpm_table[sh->mpm_stream_ctx_tc->mpm_type].Prepare(sh->mpm_stream_ctx_tc); - } - } - } - - if (sh->mpm_uri_ctx_ts != NULL) { - if (sh->mpm_uri_ctx_ts->pattern_cnt == 0) { - MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_uri_ctx_ts); - sh->mpm_uri_ctx_ts = NULL; - } else { - if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { - if (mpm_table[sh->mpm_uri_ctx_ts->mpm_type].Prepare != NULL) - mpm_table[sh->mpm_uri_ctx_ts->mpm_type].Prepare(sh->mpm_uri_ctx_ts); - } - } - } - - if (sh->mpm_hcbd_ctx_ts != NULL) { - if (sh->mpm_hcbd_ctx_ts->pattern_cnt == 0) { - MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hcbd_ctx_ts); - sh->mpm_hcbd_ctx_ts = NULL; - } else { - if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { - if (mpm_table[sh->mpm_hcbd_ctx_ts->mpm_type].Prepare != NULL) - mpm_table[sh->mpm_hcbd_ctx_ts->mpm_type].Prepare(sh->mpm_hcbd_ctx_ts); - } - } - } - - if (sh->mpm_hsbd_ctx_tc != NULL) { - if (sh->mpm_hsbd_ctx_tc->pattern_cnt == 0) { - MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hsbd_ctx_tc); - sh->mpm_hsbd_ctx_tc = NULL; - } else { - if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { - if (mpm_table[sh->mpm_hsbd_ctx_tc->mpm_type].Prepare != NULL) - mpm_table[sh->mpm_hsbd_ctx_tc->mpm_type].Prepare(sh->mpm_hsbd_ctx_tc); - } - } - } - - if (sh->mpm_smtp_filedata_ctx_ts != NULL) { - if (sh->mpm_smtp_filedata_ctx_ts->pattern_cnt == 0) { - MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_smtp_filedata_ctx_ts); - sh->mpm_smtp_filedata_ctx_ts = NULL; - } else { - if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { - if (mpm_table[sh->mpm_smtp_filedata_ctx_ts->mpm_type].Prepare != NULL) { - mpm_table[sh->mpm_smtp_filedata_ctx_ts->mpm_type].Prepare(sh->mpm_smtp_filedata_ctx_ts); - } - } - } - } - - if (sh->mpm_hhd_ctx_ts != NULL) { - if (sh->mpm_hhd_ctx_ts->pattern_cnt == 0) { - MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hhd_ctx_ts); - sh->mpm_hhd_ctx_ts = NULL; - } else { - if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { - if (mpm_table[sh->mpm_hhd_ctx_ts->mpm_type].Prepare != NULL) - mpm_table[sh->mpm_hhd_ctx_ts->mpm_type].Prepare(sh->mpm_hhd_ctx_ts); - } - } - } - if (sh->mpm_hhd_ctx_tc != NULL) { - if (sh->mpm_hhd_ctx_tc->pattern_cnt == 0) { - MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hhd_ctx_tc); - sh->mpm_hhd_ctx_tc = NULL; - } else { - if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { - if (mpm_table[sh->mpm_hhd_ctx_tc->mpm_type].Prepare != NULL) - mpm_table[sh->mpm_hhd_ctx_tc->mpm_type].Prepare(sh->mpm_hhd_ctx_tc); - } - } - } - - if (sh->mpm_hrhd_ctx_ts != NULL) { - if (sh->mpm_hrhd_ctx_ts->pattern_cnt == 0) { - MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hrhd_ctx_ts); - sh->mpm_hrhd_ctx_ts = NULL; - } else { - if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { - if (mpm_table[sh->mpm_hrhd_ctx_ts->mpm_type].Prepare != NULL) - mpm_table[sh->mpm_hrhd_ctx_ts->mpm_type].Prepare(sh->mpm_hrhd_ctx_ts); - } - } - } - if (sh->mpm_hrhd_ctx_tc != NULL) { - if (sh->mpm_hrhd_ctx_tc->pattern_cnt == 0) { - MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hrhd_ctx_tc); - sh->mpm_hrhd_ctx_tc = NULL; - } else { - if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { - if (mpm_table[sh->mpm_hrhd_ctx_tc->mpm_type].Prepare != NULL) - mpm_table[sh->mpm_hrhd_ctx_tc->mpm_type].Prepare(sh->mpm_hrhd_ctx_tc); - } - } - } - - if (sh->mpm_hmd_ctx_ts != NULL) { - if (sh->mpm_hmd_ctx_ts->pattern_cnt == 0) { - MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hmd_ctx_ts); - sh->mpm_hmd_ctx_ts = NULL; - } else { - if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { - if (mpm_table[sh->mpm_hmd_ctx_ts->mpm_type].Prepare != NULL) - mpm_table[sh->mpm_hmd_ctx_ts->mpm_type].Prepare(sh->mpm_hmd_ctx_ts); - } - } - } - - if (sh->mpm_hcd_ctx_ts != NULL) { - if (sh->mpm_hcd_ctx_ts->pattern_cnt == 0) { - MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hcd_ctx_ts); - sh->mpm_hcd_ctx_ts = NULL; - } else { - if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { - if (mpm_table[sh->mpm_hcd_ctx_ts->mpm_type].Prepare != NULL) - mpm_table[sh->mpm_hcd_ctx_ts->mpm_type].Prepare(sh->mpm_hcd_ctx_ts); - } - } - } - if (sh->mpm_hcd_ctx_tc != NULL) { - if (sh->mpm_hcd_ctx_tc->pattern_cnt == 0) { - MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hcd_ctx_tc); - sh->mpm_hcd_ctx_tc = NULL; - } else { - if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { - if (mpm_table[sh->mpm_hcd_ctx_tc->mpm_type].Prepare != NULL) - mpm_table[sh->mpm_hcd_ctx_tc->mpm_type].Prepare(sh->mpm_hcd_ctx_tc); - } - } - } - - if (sh->mpm_hrud_ctx_ts != NULL) { - if (sh->mpm_hrud_ctx_ts->pattern_cnt == 0) { - MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hrud_ctx_ts); - sh->mpm_hrud_ctx_ts = NULL; - } else { - if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { - if (mpm_table[sh->mpm_hrud_ctx_ts->mpm_type].Prepare != NULL) - mpm_table[sh->mpm_hrud_ctx_ts->mpm_type].Prepare(sh->mpm_hrud_ctx_ts); - } - } - } - - if (sh->mpm_hsmd_ctx_tc != NULL) { - if (sh->mpm_hsmd_ctx_tc->pattern_cnt == 0) { - MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hsmd_ctx_tc); - sh->mpm_hsmd_ctx_tc = NULL; - } else { - if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { - if (mpm_table[sh->mpm_hsmd_ctx_tc->mpm_type].Prepare != NULL) - mpm_table[sh->mpm_hsmd_ctx_tc->mpm_type].Prepare(sh->mpm_hsmd_ctx_tc); - } - } - } - - if (sh->mpm_hscd_ctx_tc != NULL) { - if (sh->mpm_hscd_ctx_tc->pattern_cnt == 0) { - MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hscd_ctx_tc); - sh->mpm_hscd_ctx_tc = NULL; - } else { - if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { - if (mpm_table[sh->mpm_hscd_ctx_tc->mpm_type].Prepare != NULL) - mpm_table[sh->mpm_hscd_ctx_tc->mpm_type].Prepare(sh->mpm_hscd_ctx_tc); - } - } - } - - if (sh->mpm_huad_ctx_ts != NULL) { - if (sh->mpm_huad_ctx_ts->pattern_cnt == 0) { - MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_huad_ctx_ts); - sh->mpm_huad_ctx_ts = NULL; - } else { - if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { - if (mpm_table[sh->mpm_huad_ctx_ts->mpm_type].Prepare != NULL) - mpm_table[sh->mpm_huad_ctx_ts->mpm_type].Prepare(sh->mpm_huad_ctx_ts); - } - } - } - - if (sh->mpm_hhhd_ctx_ts != NULL) { - if (sh->mpm_hhhd_ctx_ts->pattern_cnt == 0) { - MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hhhd_ctx_ts); - sh->mpm_hhhd_ctx_ts = NULL; - } else { - if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { - if (mpm_table[sh->mpm_hhhd_ctx_ts->mpm_type].Prepare != NULL) - mpm_table[sh->mpm_hhhd_ctx_ts->mpm_type].Prepare(sh->mpm_hhhd_ctx_ts); - } - } - } - - if (sh->mpm_hrhhd_ctx_ts != NULL) { - if (sh->mpm_hrhhd_ctx_ts->pattern_cnt == 0) { - MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hrhhd_ctx_ts); - sh->mpm_hrhhd_ctx_ts = NULL; - } else { - if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { - if (mpm_table[sh->mpm_hrhhd_ctx_ts->mpm_type].Prepare != NULL) - mpm_table[sh->mpm_hrhhd_ctx_ts->mpm_type].Prepare(sh->mpm_hrhhd_ctx_ts); - } - } - } - - if (sh->mpm_dnsquery_ctx_ts != NULL) { - if (sh->mpm_dnsquery_ctx_ts->pattern_cnt == 0) { - MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_dnsquery_ctx_ts); - sh->mpm_dnsquery_ctx_ts = NULL; - } else { - if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { - if (mpm_table[sh->mpm_dnsquery_ctx_ts->mpm_type].Prepare != NULL) - mpm_table[sh->mpm_dnsquery_ctx_ts->mpm_type].Prepare(sh->mpm_dnsquery_ctx_ts); - } - } - } - //} /* if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) */ + if (sh->mpm_proto_tcp_ctx_ts != NULL) { + if (sh->mpm_proto_tcp_ctx_ts->pattern_cnt == 0) { + MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_proto_tcp_ctx_ts); + sh->mpm_proto_tcp_ctx_ts = NULL; + } else { + if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { + if (mpm_table[sh->mpm_proto_tcp_ctx_ts->mpm_type].Prepare != NULL) { + mpm_table[sh->mpm_proto_tcp_ctx_ts->mpm_type]. + Prepare(sh->mpm_proto_tcp_ctx_ts); + } + } + } + } + if (sh->mpm_proto_tcp_ctx_tc != NULL) { + if (sh->mpm_proto_tcp_ctx_tc->pattern_cnt == 0) { + MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_proto_tcp_ctx_tc); + sh->mpm_proto_tcp_ctx_tc = NULL; + } else { + if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { + if (mpm_table[sh->mpm_proto_tcp_ctx_tc->mpm_type].Prepare != NULL) { + mpm_table[sh->mpm_proto_tcp_ctx_tc->mpm_type]. + Prepare(sh->mpm_proto_tcp_ctx_tc); + } + } + } + } + + if (sh->mpm_proto_udp_ctx_ts != NULL) { + if (sh->mpm_proto_udp_ctx_ts->pattern_cnt == 0) { + MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_proto_udp_ctx_ts); + sh->mpm_proto_udp_ctx_ts = NULL; + } else { + if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { + if (mpm_table[sh->mpm_proto_udp_ctx_ts->mpm_type].Prepare != NULL) { + mpm_table[sh->mpm_proto_udp_ctx_ts->mpm_type]. + Prepare(sh->mpm_proto_udp_ctx_ts); + } + } + } + } + if (sh->mpm_proto_udp_ctx_tc != NULL) { + if (sh->mpm_proto_udp_ctx_tc->pattern_cnt == 0) { + MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_proto_udp_ctx_tc); + sh->mpm_proto_udp_ctx_tc = NULL; + } else { + if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { + if (mpm_table[sh->mpm_proto_udp_ctx_tc->mpm_type].Prepare != NULL) { + mpm_table[sh->mpm_proto_udp_ctx_tc->mpm_type]. + Prepare(sh->mpm_proto_udp_ctx_tc); + } + } + } + } + + if (sh->mpm_proto_other_ctx != NULL) { + if (sh->mpm_proto_other_ctx->pattern_cnt == 0) { + MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_proto_other_ctx); + sh->mpm_proto_other_ctx = NULL; + } else { + if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { + if (mpm_table[sh->mpm_proto_other_ctx->mpm_type].Prepare != NULL) { + mpm_table[sh->mpm_proto_other_ctx->mpm_type]. + Prepare(sh->mpm_proto_other_ctx); + } + } + } + } + + if (sh->mpm_stream_ctx_ts != NULL) { + if (sh->mpm_stream_ctx_ts->pattern_cnt == 0) { + MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_stream_ctx_ts); + sh->mpm_stream_ctx_ts = NULL; + } else { + if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { + if (mpm_table[sh->mpm_stream_ctx_ts->mpm_type].Prepare != NULL) + mpm_table[sh->mpm_stream_ctx_ts->mpm_type].Prepare(sh->mpm_stream_ctx_ts); + } + } + } + if (sh->mpm_stream_ctx_tc != NULL) { + if (sh->mpm_stream_ctx_tc->pattern_cnt == 0) { + MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_stream_ctx_tc); + sh->mpm_stream_ctx_tc = NULL; + } else { + if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { + if (mpm_table[sh->mpm_stream_ctx_tc->mpm_type].Prepare != NULL) + mpm_table[sh->mpm_stream_ctx_tc->mpm_type].Prepare(sh->mpm_stream_ctx_tc); + } + } + } + + if (sh->mpm_uri_ctx_ts != NULL) { + if (sh->mpm_uri_ctx_ts->pattern_cnt == 0) { + MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_uri_ctx_ts); + sh->mpm_uri_ctx_ts = NULL; + } else { + if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { + if (mpm_table[sh->mpm_uri_ctx_ts->mpm_type].Prepare != NULL) + mpm_table[sh->mpm_uri_ctx_ts->mpm_type].Prepare(sh->mpm_uri_ctx_ts); + } + } + } + + if (sh->mpm_hcbd_ctx_ts != NULL) { + if (sh->mpm_hcbd_ctx_ts->pattern_cnt == 0) { + MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hcbd_ctx_ts); + sh->mpm_hcbd_ctx_ts = NULL; + } else { + if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { + if (mpm_table[sh->mpm_hcbd_ctx_ts->mpm_type].Prepare != NULL) + mpm_table[sh->mpm_hcbd_ctx_ts->mpm_type].Prepare(sh->mpm_hcbd_ctx_ts); + } + } + } + + if (sh->mpm_hsbd_ctx_tc != NULL) { + if (sh->mpm_hsbd_ctx_tc->pattern_cnt == 0) { + MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hsbd_ctx_tc); + sh->mpm_hsbd_ctx_tc = NULL; + } else { + if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { + if (mpm_table[sh->mpm_hsbd_ctx_tc->mpm_type].Prepare != NULL) + mpm_table[sh->mpm_hsbd_ctx_tc->mpm_type].Prepare(sh->mpm_hsbd_ctx_tc); + } + } + } + + if (sh->mpm_smtp_filedata_ctx_ts != NULL) { + if (sh->mpm_smtp_filedata_ctx_ts->pattern_cnt == 0) { + MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_smtp_filedata_ctx_ts); + sh->mpm_smtp_filedata_ctx_ts = NULL; + } else { + if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { + if (mpm_table[sh->mpm_smtp_filedata_ctx_ts->mpm_type].Prepare != NULL) { + mpm_table[sh->mpm_smtp_filedata_ctx_ts->mpm_type].Prepare(sh->mpm_smtp_filedata_ctx_ts); + } + } + } + } + + if (sh->mpm_hhd_ctx_ts != NULL) { + if (sh->mpm_hhd_ctx_ts->pattern_cnt == 0) { + MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hhd_ctx_ts); + sh->mpm_hhd_ctx_ts = NULL; + } else { + if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { + if (mpm_table[sh->mpm_hhd_ctx_ts->mpm_type].Prepare != NULL) + mpm_table[sh->mpm_hhd_ctx_ts->mpm_type].Prepare(sh->mpm_hhd_ctx_ts); + } + } + } + if (sh->mpm_hhd_ctx_tc != NULL) { + if (sh->mpm_hhd_ctx_tc->pattern_cnt == 0) { + MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hhd_ctx_tc); + sh->mpm_hhd_ctx_tc = NULL; + } else { + if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { + if (mpm_table[sh->mpm_hhd_ctx_tc->mpm_type].Prepare != NULL) + mpm_table[sh->mpm_hhd_ctx_tc->mpm_type].Prepare(sh->mpm_hhd_ctx_tc); + } + } + } + + if (sh->mpm_hrhd_ctx_ts != NULL) { + if (sh->mpm_hrhd_ctx_ts->pattern_cnt == 0) { + MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hrhd_ctx_ts); + sh->mpm_hrhd_ctx_ts = NULL; + } else { + if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { + if (mpm_table[sh->mpm_hrhd_ctx_ts->mpm_type].Prepare != NULL) + mpm_table[sh->mpm_hrhd_ctx_ts->mpm_type].Prepare(sh->mpm_hrhd_ctx_ts); + } + } + } + if (sh->mpm_hrhd_ctx_tc != NULL) { + if (sh->mpm_hrhd_ctx_tc->pattern_cnt == 0) { + MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hrhd_ctx_tc); + sh->mpm_hrhd_ctx_tc = NULL; + } else { + if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { + if (mpm_table[sh->mpm_hrhd_ctx_tc->mpm_type].Prepare != NULL) + mpm_table[sh->mpm_hrhd_ctx_tc->mpm_type].Prepare(sh->mpm_hrhd_ctx_tc); + } + } + } + + if (sh->mpm_hmd_ctx_ts != NULL) { + if (sh->mpm_hmd_ctx_ts->pattern_cnt == 0) { + MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hmd_ctx_ts); + sh->mpm_hmd_ctx_ts = NULL; + } else { + if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { + if (mpm_table[sh->mpm_hmd_ctx_ts->mpm_type].Prepare != NULL) + mpm_table[sh->mpm_hmd_ctx_ts->mpm_type].Prepare(sh->mpm_hmd_ctx_ts); + } + } + } + + if (sh->mpm_hcd_ctx_ts != NULL) { + if (sh->mpm_hcd_ctx_ts->pattern_cnt == 0) { + MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hcd_ctx_ts); + sh->mpm_hcd_ctx_ts = NULL; + } else { + if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { + if (mpm_table[sh->mpm_hcd_ctx_ts->mpm_type].Prepare != NULL) + mpm_table[sh->mpm_hcd_ctx_ts->mpm_type].Prepare(sh->mpm_hcd_ctx_ts); + } + } + } + if (sh->mpm_hcd_ctx_tc != NULL) { + if (sh->mpm_hcd_ctx_tc->pattern_cnt == 0) { + MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hcd_ctx_tc); + sh->mpm_hcd_ctx_tc = NULL; + } else { + if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { + if (mpm_table[sh->mpm_hcd_ctx_tc->mpm_type].Prepare != NULL) + mpm_table[sh->mpm_hcd_ctx_tc->mpm_type].Prepare(sh->mpm_hcd_ctx_tc); + } + } + } + + if (sh->mpm_hrud_ctx_ts != NULL) { + if (sh->mpm_hrud_ctx_ts->pattern_cnt == 0) { + MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hrud_ctx_ts); + sh->mpm_hrud_ctx_ts = NULL; + } else { + if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { + if (mpm_table[sh->mpm_hrud_ctx_ts->mpm_type].Prepare != NULL) + mpm_table[sh->mpm_hrud_ctx_ts->mpm_type].Prepare(sh->mpm_hrud_ctx_ts); + } + } + } + + if (sh->mpm_hsmd_ctx_tc != NULL) { + if (sh->mpm_hsmd_ctx_tc->pattern_cnt == 0) { + MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hsmd_ctx_tc); + sh->mpm_hsmd_ctx_tc = NULL; + } else { + if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { + if (mpm_table[sh->mpm_hsmd_ctx_tc->mpm_type].Prepare != NULL) + mpm_table[sh->mpm_hsmd_ctx_tc->mpm_type].Prepare(sh->mpm_hsmd_ctx_tc); + } + } + } + + if (sh->mpm_hscd_ctx_tc != NULL) { + if (sh->mpm_hscd_ctx_tc->pattern_cnt == 0) { + MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hscd_ctx_tc); + sh->mpm_hscd_ctx_tc = NULL; + } else { + if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { + if (mpm_table[sh->mpm_hscd_ctx_tc->mpm_type].Prepare != NULL) + mpm_table[sh->mpm_hscd_ctx_tc->mpm_type].Prepare(sh->mpm_hscd_ctx_tc); + } + } + } + + if (sh->mpm_huad_ctx_ts != NULL) { + if (sh->mpm_huad_ctx_ts->pattern_cnt == 0) { + MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_huad_ctx_ts); + sh->mpm_huad_ctx_ts = NULL; + } else { + if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { + if (mpm_table[sh->mpm_huad_ctx_ts->mpm_type].Prepare != NULL) + mpm_table[sh->mpm_huad_ctx_ts->mpm_type].Prepare(sh->mpm_huad_ctx_ts); + } + } + } + + if (sh->mpm_hhhd_ctx_ts != NULL) { + if (sh->mpm_hhhd_ctx_ts->pattern_cnt == 0) { + MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hhhd_ctx_ts); + sh->mpm_hhhd_ctx_ts = NULL; + } else { + if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { + if (mpm_table[sh->mpm_hhhd_ctx_ts->mpm_type].Prepare != NULL) + mpm_table[sh->mpm_hhhd_ctx_ts->mpm_type].Prepare(sh->mpm_hhhd_ctx_ts); + } + } + } + + if (sh->mpm_hrhhd_ctx_ts != NULL) { + if (sh->mpm_hrhhd_ctx_ts->pattern_cnt == 0) { + MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hrhhd_ctx_ts); + sh->mpm_hrhhd_ctx_ts = NULL; + } else { + if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { + if (mpm_table[sh->mpm_hrhhd_ctx_ts->mpm_type].Prepare != NULL) + mpm_table[sh->mpm_hrhhd_ctx_ts->mpm_type].Prepare(sh->mpm_hrhhd_ctx_ts); + } + } + } + + if (sh->mpm_dnsquery_ctx_ts != NULL) { + if (sh->mpm_dnsquery_ctx_ts->pattern_cnt == 0) { + MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_dnsquery_ctx_ts); + sh->mpm_dnsquery_ctx_ts = NULL; + } else { + if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { + if (mpm_table[sh->mpm_dnsquery_ctx_ts->mpm_type].Prepare != NULL) + mpm_table[sh->mpm_dnsquery_ctx_ts->mpm_type].Prepare(sh->mpm_dnsquery_ctx_ts); + } + } + } } else { MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_proto_other_ctx); sh->mpm_proto_other_ctx = NULL; diff --git a/framework/src/suricata/src/detect-engine-siggroup.c b/framework/src/suricata/src/detect-engine-siggroup.c index 02602d90..89e0eb79 100644 --- a/framework/src/suricata/src/detect-engine-siggroup.c +++ b/framework/src/suricata/src/detect-engine-siggroup.c @@ -608,7 +608,7 @@ uint32_t SigGroupHeadHashFunc(HashListTable *ht, void *data, uint16_t datalen) uint32_t hash = 0; uint32_t b = 0; - SCLogDebug("hashing sgh %p (mpm_content_maxlen %u)", sgh, sgh->mpm_content_maxlen); + SCLogDebug("hashing sgh %p (mpm_content_minlen %u)", sgh, sgh->mpm_content_minlen); for (b = 0; b < sgh->init->sig_size; b++) hash += sgh->init->sig_array[b]; @@ -998,6 +998,17 @@ void SigGroupHeadFreeMpmArrays(DetectEngineCtx *de_ctx) return; } +static uint16_t SignatureGetMpmPatternLen(Signature *s, int list) +{ + if (s->sm_lists[list] != NULL && s->mpm_sm != NULL && + SigMatchListSMBelongsTo(s, s->mpm_sm) == list) + { + DetectContentData *cd = (DetectContentData *)s->mpm_sm->ctx; + return cd->content_len; + } + return 0; +} + /** * \brief Add a Signature to a SigGroupHead. * @@ -1025,28 +1036,18 @@ int SigGroupHeadAppendSig(DetectEngineCtx *de_ctx, SigGroupHead **sgh, /* enable the sig in the bitarray */ (*sgh)->init->sig_array[s->num / 8] |= 1 << (s->num % 8); - /* update maxlen for mpm */ + /* update minlen for mpm */ if (s->sm_lists[DETECT_SM_LIST_PMATCH] != NULL) { /* check with the precalculated values from the sig */ - if (s->mpm_content_maxlen > 0) { - if ((*sgh)->mpm_content_maxlen == 0) - (*sgh)->mpm_content_maxlen = s->mpm_content_maxlen; + uint16_t mpm_content_minlen = SignatureGetMpmPatternLen(s, DETECT_SM_LIST_PMATCH); + if (mpm_content_minlen > 0) { + if ((*sgh)->mpm_content_minlen == 0) + (*sgh)->mpm_content_minlen = mpm_content_minlen; - if ((*sgh)->mpm_content_maxlen > s->mpm_content_maxlen) - (*sgh)->mpm_content_maxlen = s->mpm_content_maxlen; + if ((*sgh)->mpm_content_minlen > mpm_content_minlen) + (*sgh)->mpm_content_minlen = mpm_content_minlen; - SCLogDebug("(%p)->mpm_content_maxlen %u", *sgh, (*sgh)->mpm_content_maxlen); - } - } - if (s->sm_lists[DETECT_SM_LIST_UMATCH] != NULL) { - if (s->mpm_uricontent_maxlen > 0) { - if ((*sgh)->mpm_uricontent_maxlen == 0) - (*sgh)->mpm_uricontent_maxlen = s->mpm_uricontent_maxlen; - - if ((*sgh)->mpm_uricontent_maxlen > s->mpm_uricontent_maxlen) - (*sgh)->mpm_uricontent_maxlen = s->mpm_uricontent_maxlen; - - SCLogDebug("(%p)->mpm_uricontent_maxlen %u", *sgh, (*sgh)->mpm_uricontent_maxlen); + SCLogDebug("(%p)->mpm_content_minlen %u", *sgh, (*sgh)->mpm_content_minlen); } } return 0; @@ -1103,23 +1104,16 @@ int SigGroupHeadCopySigs(DetectEngineCtx *de_ctx, SigGroupHead *src, SigGroupHea for (idx = 0; idx < src->init->sig_size; idx++) (*dst)->init->sig_array[idx] = (*dst)->init->sig_array[idx] | src->init->sig_array[idx]; - if (src->mpm_content_maxlen != 0) { - if ((*dst)->mpm_content_maxlen == 0) - (*dst)->mpm_content_maxlen = src->mpm_content_maxlen; - - if ((*dst)->mpm_content_maxlen > src->mpm_content_maxlen) - (*dst)->mpm_content_maxlen = src->mpm_content_maxlen; + if (src->mpm_content_minlen != 0) { + if ((*dst)->mpm_content_minlen == 0) + (*dst)->mpm_content_minlen = src->mpm_content_minlen; - SCLogDebug("src (%p)->mpm_content_maxlen %u", src, src->mpm_content_maxlen); - SCLogDebug("dst (%p)->mpm_content_maxlen %u", (*dst), (*dst)->mpm_content_maxlen); - BUG_ON((*dst)->mpm_content_maxlen == 0); - } - if (src->mpm_uricontent_maxlen != 0) { - if ((*dst)->mpm_uricontent_maxlen == 0) - (*dst)->mpm_uricontent_maxlen = src->mpm_uricontent_maxlen; + if ((*dst)->mpm_content_minlen > src->mpm_content_minlen) + (*dst)->mpm_content_minlen = src->mpm_content_minlen; - if ((*dst)->mpm_uricontent_maxlen > src->mpm_uricontent_maxlen) - (*dst)->mpm_uricontent_maxlen = src->mpm_uricontent_maxlen; + SCLogDebug("src (%p)->mpm_content_minlen %u", src, src->mpm_content_minlen); + SCLogDebug("dst (%p)->mpm_content_minlen %u", (*dst), (*dst)->mpm_content_minlen); + BUG_ON((*dst)->mpm_content_minlen == 0); } return 0; @@ -1606,6 +1600,42 @@ void SigGroupHeadSetFilemagicFlag(DetectEngineCtx *de_ctx, SigGroupHead *sgh) } /** + * \brief Get size of the shortest mpm pattern. + * + * \param de_ctx detection engine ctx for the signatures + * \param sgh sig group head to set the flag in + * \param list sm_list to consider + */ +uint16_t SigGroupHeadGetMinMpmSize(DetectEngineCtx *de_ctx, + SigGroupHead *sgh, int list) +{ + Signature *s = NULL; + uint32_t sig = 0; + uint16_t min = USHRT_MAX; + + if (sgh == NULL) + return 0; + + for (sig = 0; sig < sgh->sig_cnt; sig++) { + s = sgh->match_array[sig]; + if (s == NULL) + continue; + + uint16_t mpm_content_minlen = SignatureGetMpmPatternLen(s, DETECT_SM_LIST_PMATCH); + if (mpm_content_minlen > 0) { + if (mpm_content_minlen < min) + min = mpm_content_minlen; + SCLogDebug("mpm_content_minlen %u", mpm_content_minlen); + } + } + + if (min == USHRT_MAX) + min = 0; + SCLogDebug("min mpm size %u", min); + return min; +} + +/** * \brief Set the need size flag in the sgh. * * \param de_ctx detection engine ctx for the signatures diff --git a/framework/src/suricata/src/detect-engine-siggroup.h b/framework/src/suricata/src/detect-engine-siggroup.h index 829b0cef..cd6810a1 100644 --- a/framework/src/suricata/src/detect-engine-siggroup.h +++ b/framework/src/suricata/src/detect-engine-siggroup.h @@ -88,6 +88,8 @@ void SigGroupHeadSetFilemagicFlag(DetectEngineCtx *, SigGroupHead *); void SigGroupHeadSetFilestoreCount(DetectEngineCtx *, SigGroupHead *); void SigGroupHeadSetFileMd5Flag(DetectEngineCtx *, SigGroupHead *); void SigGroupHeadSetFilesizeFlag(DetectEngineCtx *, SigGroupHead *); +uint16_t SigGroupHeadGetMinMpmSize(DetectEngineCtx *de_ctx, + SigGroupHead *sgh, int list); int SigGroupHeadBuildNonMpmArray(DetectEngineCtx *de_ctx, SigGroupHead *sgh); diff --git a/framework/src/suricata/src/detect-engine-threshold.c b/framework/src/suricata/src/detect-engine-threshold.c index 6c6d1371..2d37e55e 100644 --- a/framework/src/suricata/src/detect-engine-threshold.c +++ b/framework/src/suricata/src/detect-engine-threshold.c @@ -216,11 +216,11 @@ int ThresholdHandlePacketSuppress(Packet *p, DetectThresholdData *td, uint32_t s switch (td->track) { case TRACK_DST: m = DetectAddressLookupInHead(&td->addrs, &p->dst); - SCLogInfo("TRACK_DST"); + SCLogDebug("TRACK_DST"); break; case TRACK_SRC: m = DetectAddressLookupInHead(&td->addrs, &p->src); - SCLogInfo("TRACK_SRC"); + SCLogDebug("TRACK_SRC"); break; /* suppress if either src or dst is a match on the suppress * address list */ diff --git a/framework/src/suricata/src/detect-engine.c b/framework/src/suricata/src/detect-engine.c index c6e1a83f..431f4b2a 100644 --- a/framework/src/suricata/src/detect-engine.c +++ b/framework/src/suricata/src/detect-engine.c @@ -2340,7 +2340,7 @@ static int reloads = 0; * \retval -1 error * \retval 0 ok */ -int DetectEngineReload(const char *filename) +int DetectEngineReload(const char *filename, SCInstance *suri) { DetectEngineCtx *new_de_ctx = NULL; DetectEngineCtx *old_de_ctx = NULL; @@ -2377,7 +2377,8 @@ int DetectEngineReload(const char *filename) DetectEngineDeReference(&old_de_ctx); return -1; } - if (SigLoadSignatures(new_de_ctx, NULL, 0) != 0) { + if (SigLoadSignatures(new_de_ctx, + suri->sig_file, suri->sig_file_exclusive) != 0) { DetectEngineCtxFree(new_de_ctx); DetectEngineDeReference(&old_de_ctx); return -1; diff --git a/framework/src/suricata/src/detect-engine.h b/framework/src/suricata/src/detect-engine.h index ce486213..7b621710 100644 --- a/framework/src/suricata/src/detect-engine.h +++ b/framework/src/suricata/src/detect-engine.h @@ -75,7 +75,7 @@ void DetectEnginePruneFreeList(void); int DetectEngineMoveToFreeList(DetectEngineCtx *de_ctx); DetectEngineCtx *DetectEngineReference(DetectEngineCtx *); void DetectEngineDeReference(DetectEngineCtx **de_ctx); -int DetectEngineReload(const char *filename); +int DetectEngineReload(const char *filename, SCInstance *suri); int DetectEngineEnabled(void); int DetectEngineMTApply(void); int DetectEngineMultiTenantEnabled(void); diff --git a/framework/src/suricata/src/detect-lua-extensions.c b/framework/src/suricata/src/detect-lua-extensions.c index 020c886d..ae9b4e14 100644 --- a/framework/src/suricata/src/detect-lua-extensions.c +++ b/framework/src/suricata/src/detect-lua-extensions.c @@ -67,6 +67,7 @@ #include "util-lua-http.h" #include "util-lua-dns.h" #include "util-lua-tls.h" +#include "util-lua-ssh.h" static const char luaext_key_ld[] = "suricata:luajitdata"; static const char luaext_key_det_ctx[] = "suricata:det_ctx"; @@ -619,6 +620,7 @@ int LuaRegisterExtensions(lua_State *lua_state) LuaRegisterHttpFunctions(lua_state); LuaRegisterDnsFunctions(lua_state); LuaRegisterTlsFunctions(lua_state); + LuaRegisterSshFunctions(lua_state); return 0; } diff --git a/framework/src/suricata/src/detect-lua.c b/framework/src/suricata/src/detect-lua.c index d1f2cd00..a3eead4a 100644 --- a/framework/src/suricata/src/detect-lua.c +++ b/framework/src/suricata/src/detect-lua.c @@ -166,6 +166,8 @@ void DetectLuaRegister(void) #define DATATYPE_TLS (1<<18) +#define DATATYPE_SSH (1<<19) + #ifdef HAVE_LUAJIT static void *LuaStatePoolAlloc(void) { @@ -190,12 +192,18 @@ int DetectLuajitSetupStatesPool(int num, int reloads) pthread_mutex_lock(&luajit_states_lock); if (luajit_states == NULL) { - int cnt = 0; - char *conf_val = NULL; + intmax_t cnt = 0; + ConfNode *denode = NULL; + ConfNode *decnf = ConfGetNode("detect-engine"); + if (decnf != NULL) { + TAILQ_FOREACH(denode, &decnf->head, next) { + if (strcmp(denode->val, "luajit-states") == 0) { + ConfGetChildValueInt(denode, "luajit-states", &cnt); + } + } + } - if ((ConfGet("detect-engine.luajit-states", &conf_val)) == 1) { - cnt = (int)atoi(conf_val); - } else { + if (cnt == 0) { int cpus = UtilCpuGetNumProcessorsOnline(); if (cpus == 0) { cpus = 10; @@ -1008,6 +1016,12 @@ static int DetectLuaSetupPrime(DetectEngineCtx *de_ctx, DetectLuaData *ld) ld->flags |= DATATYPE_TLS; + } else if (strncmp(k, "ssh", 3) == 0 && strcmp(v, "true") == 0) { + + ld->alproto = ALPROTO_SSH; + + ld->flags |= DATATYPE_SSH; + } else { SCLogError(SC_ERR_LUA_ERROR, "unsupported data type %s", k); goto error; @@ -1105,6 +1119,8 @@ static int DetectLuaSetup (DetectEngineCtx *de_ctx, Signature *s, char *str) } } else if (luajit->alproto == ALPROTO_TLS) { SigMatchAppendSMToList(s, sm, DETECT_SM_LIST_AMATCH); + } else if (luajit->alproto == ALPROTO_SSH) { + SigMatchAppendSMToList(s, sm, DETECT_SM_LIST_AMATCH); } else { SCLogError(SC_ERR_LUA_ERROR, "luajit can't be used with protocol %s", AppLayerGetProtoName(luajit->alproto)); diff --git a/framework/src/suricata/src/detect-parse.c b/framework/src/suricata/src/detect-parse.c index fedfebe8..9ac53e03 100644 --- a/framework/src/suricata/src/detect-parse.c +++ b/framework/src/suricata/src/detect-parse.c @@ -1345,7 +1345,6 @@ int SigValidate(DetectEngineCtx *de_ctx, Signature *s) static Signature *SigInitHelper(DetectEngineCtx *de_ctx, char *sigstr, uint8_t dir) { - SigMatch *sm; Signature *sig = SigAlloc(); if (sig == NULL) goto error; @@ -1390,42 +1389,6 @@ static Signature *SigInitHelper(DetectEngineCtx *de_ctx, char *sigstr, if (DetectAppLayerEventPrepare(sig) < 0) goto error; - /* set mpm_content_len */ - - /* determine the length of the longest pattern in the sig */ - if (sig->sm_lists[DETECT_SM_LIST_PMATCH] != NULL) { - sig->mpm_content_maxlen = 0; - - for (sm = sig->sm_lists[DETECT_SM_LIST_PMATCH]; sm != NULL; sm = sm->next) { - if (sm->type == DETECT_CONTENT) { - DetectContentData *cd = (DetectContentData *)sm->ctx; - if (cd == NULL) - continue; - - if (sig->mpm_content_maxlen == 0) - sig->mpm_content_maxlen = cd->content_len; - if (sig->mpm_content_maxlen < cd->content_len) - sig->mpm_content_maxlen = cd->content_len; - } - } - } - if (sig->sm_lists[DETECT_SM_LIST_UMATCH] != NULL) { - sig->mpm_uricontent_maxlen = 0; - - for (sm = sig->sm_lists[DETECT_SM_LIST_UMATCH]; sm != NULL; sm = sm->next) { - if (sm->type == DETECT_CONTENT) { - DetectContentData *ud = (DetectContentData *)sm->ctx; - if (ud == NULL) - continue; - - if (sig->mpm_uricontent_maxlen == 0) - sig->mpm_uricontent_maxlen = ud->content_len; - if (sig->mpm_uricontent_maxlen < ud->content_len) - sig->mpm_uricontent_maxlen = ud->content_len; - } - } - } - /* set the packet and app layer flags, but only if the * app layer flag wasn't already set in which case we * only consider the app layer */ @@ -3268,16 +3231,6 @@ int SigParseTestMpm01 (void) goto end; } - if (sig->mpm_content_maxlen != 4) { - printf("mpm content max len %"PRIu16", expected 4: ", sig->mpm_content_maxlen); - goto end; - } - - if (sig->mpm_uricontent_maxlen != 0) { - printf("mpm uricontent max len %"PRIu16", expected 0: ", sig->mpm_uricontent_maxlen); - goto end; - } - result = 1; end: if (sig != NULL) @@ -3309,16 +3262,6 @@ int SigParseTestMpm02 (void) goto end; } - if (sig->mpm_content_maxlen != 6) { - printf("mpm content max len %"PRIu16", expected 6: ", sig->mpm_content_maxlen); - goto end; - } - - if (sig->mpm_uricontent_maxlen != 0) { - printf("mpm uricontent max len %"PRIu16", expected 0: ", sig->mpm_uricontent_maxlen); - goto end; - } - result = 1; end: if (sig != NULL) diff --git a/framework/src/suricata/src/detect-pcre.c b/framework/src/suricata/src/detect-pcre.c index 86f3d169..d69f1da6 100644 --- a/framework/src/suricata/src/detect-pcre.c +++ b/framework/src/suricata/src/detect-pcre.c @@ -274,6 +274,26 @@ static int DetectPcreSetList(int list, int set) return set; } +static int DetectPcreHasUpperCase(const char *re) +{ + size_t len = strlen(re); + int is_meta = 0; + + for (size_t i = 0; i < len; i++) { + if (is_meta) { + is_meta = 0; + } + else if (re[i] == '\\') { + is_meta = 1; + } + else if (isupper((unsigned char)re[i])) { + return 1; + } + } + + return 0; +} + static DetectPcreData *DetectPcreParse (DetectEngineCtx *de_ctx, char *regexstr, int *sm_list) { int ec; @@ -290,8 +310,6 @@ static DetectPcreData *DetectPcreParse (DetectEngineCtx *de_ctx, char *regexstr, char re[slen], op_str[64] = ""; uint16_t pos = 0; uint8_t negate = 0; - uint16_t re_len = 0; - uint32_t u = 0; while (pos < slen && isspace((unsigned char)regexstr[pos])) { pos++; @@ -476,18 +494,14 @@ static DetectPcreData *DetectPcreParse (DetectEngineCtx *de_ctx, char *regexstr, "Since the hostname buffer we match against " "is actually lowercase, having a " "nocase is redundant."); - } else { - re_len = strlen(re); - for (u = 0; u < re_len; u++) { - if (isupper((unsigned char)re[u])) { - SCLogError(SC_ERR_INVALID_SIGNATURE, "pcre host(\"W\") " - "specified has an uppercase char. " - "Since the hostname buffer we match against " - "is actually lowercase, please specify an " - "all lowercase based pcre."); - goto error; - } - } + } + else if (DetectPcreHasUpperCase(re)) { + SCLogError(SC_ERR_INVALID_SIGNATURE, "pcre host(\"W\") " + "specified has an uppercase char. " + "Since the hostname buffer we match against " + "is actually lowercase, please specify an " + "all lowercase based pcre."); + goto error; } } @@ -4053,6 +4067,57 @@ end: return result; } +/** + * \brief Test parsing of pcre's with the W modifier set. + */ +static int DetectPcreParseHttpHost(void) +{ + int result = 0; + DetectPcreData *pd = NULL; + int list = DETECT_SM_LIST_NOTSET; + DetectEngineCtx *de_ctx = DetectEngineCtxInit(); + + if (de_ctx == NULL) { + return 0; + } + + pd = DetectPcreParse(de_ctx, "/domain\\.com/W", &list); + if (pd == NULL) { + goto end; + } + DetectPcreFree(pd); + + list = DETECT_SM_LIST_NOTSET; + pd = DetectPcreParse(de_ctx, "/dOmain\\.com/W", &list); + if (pd != NULL) { + DetectPcreFree(pd); + goto end; + } + + /* Uppercase meta characters are valid. */ + list = DETECT_SM_LIST_NOTSET; + pd = DetectPcreParse(de_ctx, "/domain\\D+\\.com/W", &list); + if (pd == NULL) { + goto end; + } + DetectPcreFree(pd); + + /* This should not parse as the first \ escapes the second \, then + * we have a D. */ + list = DETECT_SM_LIST_NOTSET; + pd = DetectPcreParse(de_ctx, "/\\\\Ddomain\\.com/W", &list); + if (pd != NULL) { + DetectPcreFree(pd); + goto end; + } + + result = 1; + +end: + DetectEngineCtxFree(de_ctx); + return result; +} + #endif /* UNITTESTS */ /** @@ -4121,6 +4186,8 @@ void DetectPcreRegisterTests(void) UtRegisterTest("DetectPcreFlowvarCapture02 -- capture for http_header", DetectPcreFlowvarCapture02, 1); UtRegisterTest("DetectPcreFlowvarCapture03 -- capture for http_header", DetectPcreFlowvarCapture03, 1); + UtRegisterTest("DetectPcreParseHttpHost", DetectPcreParseHttpHost, 1); + #endif /* UNITTESTS */ } diff --git a/framework/src/suricata/src/detect-template-buffer.c b/framework/src/suricata/src/detect-template-buffer.c index 64f948c2..d9f9aa67 100644 --- a/framework/src/suricata/src/detect-template-buffer.c +++ b/framework/src/suricata/src/detect-template-buffer.c @@ -21,6 +21,7 @@ */ #include "suricata-common.h" +#include "conf.h" #include "detect.h" #include "app-layer-template.h" @@ -29,6 +30,10 @@ static void DetectTemplateBufferRegisterTests(void); void DetectTemplateBufferRegister(void) { + if (ConfGetNode("app-layer.protocols.template") == NULL) { + return; + } + sigmatch_table[DETECT_AL_TEMPLATE_BUFFER].name = "template_buffer"; sigmatch_table[DETECT_AL_TEMPLATE_BUFFER].desc = "Template content modififier to match on the template buffers"; diff --git a/framework/src/suricata/src/detect-uricontent.c b/framework/src/suricata/src/detect-uricontent.c index a22479b0..65d2ef94 100644 --- a/framework/src/suricata/src/detect-uricontent.c +++ b/framework/src/suricata/src/detect-uricontent.c @@ -216,13 +216,13 @@ static inline int DoDetectAppLayerUricontentMatch (DetectEngineThreadCtx *det_ct { int ret = 0; /* run the pattern matcher against the uri */ - if (det_ctx->sgh->mpm_uricontent_maxlen > uri_len) { - SCLogDebug("not searching as pkt payload is smaller than the " - "largest uricontent length we need to match"); + if (det_ctx->sgh->mpm_uricontent_minlen > uri_len) { + SCLogDebug("not searching as uri len is smaller than the " + "shortest uricontent length we need to match"); } else { - SCLogDebug("search: (%p, maxlen %" PRIu32 ", sgh->sig_cnt " - "%" PRIu32 ")", det_ctx->sgh, det_ctx->sgh-> - mpm_uricontent_maxlen, det_ctx->sgh->sig_cnt); + SCLogDebug("search: (%p, minlen %" PRIu32 ", sgh->sig_cnt " + "%" PRIu32 ")", det_ctx->sgh, + det_ctx->sgh->mpm_uricontent_minlen, det_ctx->sgh->sig_cnt); ret += UriPatternSearch(det_ctx, uri, uri_len, flags); diff --git a/framework/src/suricata/src/detect.c b/framework/src/suricata/src/detect.c index e0b5bfd3..c9a16ead 100644 --- a/framework/src/suricata/src/detect.c +++ b/framework/src/suricata/src/detect.c @@ -692,7 +692,7 @@ static StreamMsg *SigMatchSignaturesGetSmsg(Flow *f, Packet *p, uint8_t flags) /* if the smsg is bigger than the current packet, we will * process the smsg in a later run */ - if ((head->seq + head->data_len) > (TCP_GET_SEQ(p) + p->payload_len)) { + if (SEQ_GT((head->seq + head->data_len), (TCP_GET_SEQ(p) + p->payload_len))) { SCLogDebug("smsg ends beyond current packet, skipping for now %"PRIu32">%"PRIu32, (head->seq + head->data_len), (TCP_GET_SEQ(p) + p->payload_len)); goto end; @@ -711,7 +711,7 @@ static StreamMsg *SigMatchSignaturesGetSmsg(Flow *f, Packet *p, uint8_t flags) /* if the smsg is bigger than the current packet, we will * process the smsg in a later run */ - if ((head->seq + head->data_len) > (TCP_GET_SEQ(p) + p->payload_len)) { + if (SEQ_GT((head->seq + head->data_len), (TCP_GET_SEQ(p) + p->payload_len))) { SCLogDebug("smsg ends beyond current packet, skipping for now %"PRIu32">%"PRIu32, (head->seq + head->data_len), (TCP_GET_SEQ(p) + p->payload_len)); goto end; @@ -1086,8 +1086,8 @@ static inline void DetectMpmPrefilter(DetectEngineCtx *de_ctx, } if (det_ctx->sgh->flags & SIG_GROUP_HEAD_MPM_PACKET) { /* run the multi packet matcher against the payload of the packet */ - SCLogDebug("search: (%p, maxlen %" PRIu32 ", sgh->sig_cnt %" PRIu32 ")", - det_ctx->sgh, det_ctx->sgh->mpm_content_maxlen, det_ctx->sgh->sig_cnt); + SCLogDebug("search: (%p, minlen %" PRIu32 ", sgh->sig_cnt %" PRIu32 ")", + det_ctx->sgh, det_ctx->sgh->mpm_content_minlen, det_ctx->sgh->sig_cnt); PACKET_PROFILING_DETECT_START(p, PROF_DETECT_MPM_PACKET); PacketPatternSearch(det_ctx, p); @@ -3105,15 +3105,15 @@ int CreateGroupedAddrListCmpCnt(DetectAddress *a, DetectAddress *b) return 0; } -int CreateGroupedAddrListCmpMpmMaxlen(DetectAddress *a, DetectAddress *b) +int CreateGroupedAddrListCmpMpmMinlen(DetectAddress *a, DetectAddress *b) { if (a->sh == NULL || b->sh == NULL) return 0; - if (SMALL_MPM(a->sh->mpm_content_maxlen)) + if (SMALL_MPM(a->sh->mpm_content_minlen)) return 1; - if (a->sh->mpm_content_maxlen < b->sh->mpm_content_maxlen) + if (a->sh->mpm_content_minlen < b->sh->mpm_content_minlen) return 1; return 0; } @@ -3139,7 +3139,7 @@ int CreateGroupedAddrList(DetectEngineCtx *de_ctx, DetectAddress *srchead, for (gr = srchead; gr != NULL; gr = gr->next) { BUG_ON(gr->ip.family == 0 && !(gr->flags & ADDRESS_FLAG_ANY)); - if (SMALL_MPM(gr->sh->mpm_content_maxlen) && unique_groups > 0) + if (SMALL_MPM(gr->sh->mpm_content_minlen) && unique_groups > 0) unique_groups++; groups++; @@ -3280,15 +3280,15 @@ int CreateGroupedPortListCmpCnt(DetectPort *a, DetectPort *b) return 0; } -int CreateGroupedPortListCmpMpmMaxlen(DetectPort *a, DetectPort *b) +int CreateGroupedPortListCmpMpmMinlen(DetectPort *a, DetectPort *b) { if (a->sh == NULL || b->sh == NULL) return 0; - if (SMALL_MPM(a->sh->mpm_content_maxlen)) + if (SMALL_MPM(a->sh->mpm_content_minlen)) return 1; - if (a->sh->mpm_content_maxlen < b->sh->mpm_content_maxlen) + if (a->sh->mpm_content_minlen < b->sh->mpm_content_minlen) return 1; return 0; @@ -3315,7 +3315,7 @@ int CreateGroupedPortList(DetectEngineCtx *de_ctx,HashListTable *port_hash, Dete SCLogDebug("hash list gr %p", gr); DetectPortPrint(gr); - if (SMALL_MPM(gr->sh->mpm_content_maxlen) && unique_groups > 0) + if (SMALL_MPM(gr->sh->mpm_content_minlen) && unique_groups > 0) unique_groups++; groups++; @@ -3505,16 +3505,16 @@ int SigAddressPrepareStage2(DetectEngineCtx *de_ctx) CreateGroupedAddrList(de_ctx, de_ctx->flow_gh[f].tmp_gh[proto]->ipv4_head, AF_INET, de_ctx->flow_gh[f].src_gh[proto], groups, - CreateGroupedAddrListCmpMpmMaxlen, DetectEngineGetMaxSigId(de_ctx)); + CreateGroupedAddrListCmpMpmMinlen, DetectEngineGetMaxSigId(de_ctx)); CreateGroupedAddrList(de_ctx, de_ctx->flow_gh[f].tmp_gh[proto]->ipv6_head, AF_INET6, de_ctx->flow_gh[f].src_gh[proto], groups, - CreateGroupedAddrListCmpMpmMaxlen, DetectEngineGetMaxSigId(de_ctx)); + CreateGroupedAddrListCmpMpmMinlen, DetectEngineGetMaxSigId(de_ctx)); CreateGroupedAddrList(de_ctx, de_ctx->flow_gh[f].tmp_gh[proto]->any_head, AF_UNSPEC, de_ctx->flow_gh[f].src_gh[proto], groups, - CreateGroupedAddrListCmpMpmMaxlen, DetectEngineGetMaxSigId(de_ctx)); + CreateGroupedAddrListCmpMpmMinlen, DetectEngineGetMaxSigId(de_ctx)); DetectAddressHeadFree(de_ctx->flow_gh[f].tmp_gh[proto]); de_ctx->flow_gh[f].tmp_gh[proto] = NULL; @@ -3697,7 +3697,8 @@ int BuildDestinationAddressHeads(DetectEngineCtx *de_ctx, DetectAddressHead *hea * mind the limits we use. */ int groups = (flow ? de_ctx->max_uniq_toserver_dst_groups : de_ctx->max_uniq_toclient_dst_groups); - CreateGroupedAddrList(de_ctx, tmp_gr_list, family, gr->dst_gh, groups, CreateGroupedAddrListCmpMpmMaxlen, max_idx); + CreateGroupedAddrList(de_ctx, tmp_gr_list, family, gr->dst_gh, groups, + CreateGroupedAddrListCmpMpmMinlen, max_idx); /* see if the sig group head of each address group is the * same as an earlier one. If it is, free our head and use @@ -3722,62 +3723,6 @@ int BuildDestinationAddressHeads(DetectEngineCtx *de_ctx, DetectAddressHead *hea printf("PatternMatchPrepareGroup failed\n"); goto error; } - if (sgr->sh->mpm_proto_tcp_ctx_ts != NULL) { - if (de_ctx->mpm_max_patcnt < sgr->sh->mpm_proto_tcp_ctx_ts->pattern_cnt) - de_ctx->mpm_max_patcnt = sgr->sh->mpm_proto_tcp_ctx_ts->pattern_cnt; - - de_ctx->mpm_tot_patcnt += sgr->sh->mpm_proto_tcp_ctx_ts->pattern_cnt; - } - if (sgr->sh->mpm_proto_tcp_ctx_tc != NULL) { - if (de_ctx->mpm_max_patcnt < sgr->sh->mpm_proto_tcp_ctx_tc->pattern_cnt) - de_ctx->mpm_max_patcnt = sgr->sh->mpm_proto_tcp_ctx_tc->pattern_cnt; - - de_ctx->mpm_tot_patcnt += sgr->sh->mpm_proto_tcp_ctx_tc->pattern_cnt; - } - if (sgr->sh->mpm_proto_udp_ctx_ts != NULL) { - if (de_ctx->mpm_max_patcnt < sgr->sh->mpm_proto_udp_ctx_ts->pattern_cnt) - de_ctx->mpm_max_patcnt = sgr->sh->mpm_proto_udp_ctx_ts->pattern_cnt; - - de_ctx->mpm_tot_patcnt += sgr->sh->mpm_proto_udp_ctx_ts->pattern_cnt; - } - if (sgr->sh->mpm_proto_udp_ctx_tc != NULL) { - if (de_ctx->mpm_max_patcnt < sgr->sh->mpm_proto_udp_ctx_tc->pattern_cnt) - de_ctx->mpm_max_patcnt = sgr->sh->mpm_proto_udp_ctx_tc->pattern_cnt; - - de_ctx->mpm_tot_patcnt += sgr->sh->mpm_proto_udp_ctx_tc->pattern_cnt; - } - if (sgr->sh->mpm_proto_other_ctx != NULL) { - if (de_ctx->mpm_max_patcnt < sgr->sh->mpm_proto_other_ctx->pattern_cnt) - de_ctx->mpm_max_patcnt = sgr->sh->mpm_proto_other_ctx->pattern_cnt; - - de_ctx->mpm_tot_patcnt += sgr->sh->mpm_proto_other_ctx->pattern_cnt; - } - if (sgr->sh->mpm_uri_ctx_ts != NULL) { - if (de_ctx->mpm_uri_max_patcnt < sgr->sh->mpm_uri_ctx_ts->pattern_cnt) - de_ctx->mpm_uri_max_patcnt = sgr->sh->mpm_uri_ctx_ts->pattern_cnt; - - de_ctx->mpm_uri_tot_patcnt += sgr->sh->mpm_uri_ctx_ts->pattern_cnt; - } - /* dbg */ - if (!(sgr->sh->flags & SIG_GROUP_HEAD_MPM_COPY) && sgr->sh->mpm_proto_tcp_ctx_ts) { - de_ctx->mpm_memory_size += sgr->sh->mpm_proto_tcp_ctx_ts->memory_size; - } - if (!(sgr->sh->flags & SIG_GROUP_HEAD_MPM_COPY) && sgr->sh->mpm_proto_tcp_ctx_tc) { - de_ctx->mpm_memory_size += sgr->sh->mpm_proto_tcp_ctx_tc->memory_size; - } - if (!(sgr->sh->flags & SIG_GROUP_HEAD_MPM_COPY) && sgr->sh->mpm_proto_udp_ctx_ts) { - de_ctx->mpm_memory_size += sgr->sh->mpm_proto_udp_ctx_ts->memory_size; - } - if (!(sgr->sh->flags & SIG_GROUP_HEAD_MPM_COPY) && sgr->sh->mpm_proto_udp_ctx_tc) { - de_ctx->mpm_memory_size += sgr->sh->mpm_proto_udp_ctx_tc->memory_size; - } - if (!(sgr->sh->flags & SIG_GROUP_HEAD_MPM_COPY) && sgr->sh->mpm_proto_other_ctx) { - de_ctx->mpm_memory_size += sgr->sh->mpm_proto_other_ctx->memory_size; - } - if (!(sgr->sh->flags & SIG_GROUP_HEAD_MPM_URI_COPY) && sgr->sh->mpm_uri_ctx_ts) { - de_ctx->mpm_memory_size += sgr->sh->mpm_uri_ctx_ts->memory_size; - } - SigGroupHeadHashAdd(de_ctx, sgr->sh); SigGroupHeadStore(de_ctx, sgr->sh); de_ctx->gh_unique++; @@ -3871,7 +3816,8 @@ int BuildDestinationAddressHeadsWithBothPorts(DetectEngineCtx *de_ctx, DetectAdd * mind the limits we use. */ int groups = (flow ? de_ctx->max_uniq_toserver_dst_groups : de_ctx->max_uniq_toclient_dst_groups); - CreateGroupedAddrList(de_ctx, tmp_gr_list, family, src_gr->dst_gh, groups, CreateGroupedAddrListCmpMpmMaxlen, max_idx); + CreateGroupedAddrList(de_ctx, tmp_gr_list, family, src_gr->dst_gh, groups, + CreateGroupedAddrListCmpMpmMinlen, max_idx); /* add the ports to the dst address groups and the sigs * to the ports */ @@ -3922,7 +3868,8 @@ int BuildDestinationAddressHeadsWithBothPorts(DetectEngineCtx *de_ctx, DetectAdd int spgroups = (flow ? de_ctx->max_uniq_toserver_sp_groups : de_ctx->max_uniq_toclient_sp_groups); - CreateGroupedPortList(de_ctx, de_ctx->sport_hash_table, &dst_gr->port, spgroups, CreateGroupedPortListCmpMpmMaxlen, max_idx); + CreateGroupedPortList(de_ctx, de_ctx->sport_hash_table, &dst_gr->port, spgroups, + CreateGroupedPortListCmpMpmMinlen, max_idx); SCLogDebug("adding sgh %p to the hash", dst_gr->sh); SigGroupHeadHashAdd(de_ctx, dst_gr->sh); @@ -3978,7 +3925,7 @@ int BuildDestinationAddressHeadsWithBothPorts(DetectEngineCtx *de_ctx, DetectAdd CreateGroupedPortList(de_ctx, de_ctx->dport_hash_table, &sp->dst_ph, dpgroups, - CreateGroupedPortListCmpMpmMaxlen, max_idx); + CreateGroupedPortListCmpMpmMinlen, max_idx); SigGroupHeadSPortHashAdd(de_ctx, sp->sh); @@ -4010,62 +3957,6 @@ int BuildDestinationAddressHeadsWithBothPorts(DetectEngineCtx *de_ctx, DetectAdd printf("PatternMatchPrepareGroup failed\n"); goto error; } - if (dp->sh->mpm_proto_tcp_ctx_ts != NULL) { - if (de_ctx->mpm_max_patcnt < dp->sh->mpm_proto_tcp_ctx_ts->pattern_cnt) - de_ctx->mpm_max_patcnt = dp->sh->mpm_proto_tcp_ctx_ts->pattern_cnt; - - de_ctx->mpm_tot_patcnt += dp->sh->mpm_proto_tcp_ctx_ts->pattern_cnt; - } - if (dp->sh->mpm_proto_tcp_ctx_tc != NULL) { - if (de_ctx->mpm_max_patcnt < dp->sh->mpm_proto_tcp_ctx_tc->pattern_cnt) - de_ctx->mpm_max_patcnt = dp->sh->mpm_proto_tcp_ctx_tc->pattern_cnt; - - de_ctx->mpm_tot_patcnt += dp->sh->mpm_proto_tcp_ctx_tc->pattern_cnt; - } - if (dp->sh->mpm_proto_udp_ctx_ts != NULL) { - if (de_ctx->mpm_max_patcnt < dp->sh->mpm_proto_udp_ctx_ts->pattern_cnt) - de_ctx->mpm_max_patcnt = dp->sh->mpm_proto_udp_ctx_ts->pattern_cnt; - - de_ctx->mpm_tot_patcnt += dp->sh->mpm_proto_udp_ctx_ts->pattern_cnt; - } - if (dp->sh->mpm_proto_udp_ctx_tc != NULL) { - if (de_ctx->mpm_max_patcnt < dp->sh->mpm_proto_udp_ctx_tc->pattern_cnt) - de_ctx->mpm_max_patcnt = dp->sh->mpm_proto_udp_ctx_tc->pattern_cnt; - - de_ctx->mpm_tot_patcnt += dp->sh->mpm_proto_udp_ctx_tc->pattern_cnt; - } - if (dp->sh->mpm_proto_other_ctx != NULL) { - if (de_ctx->mpm_max_patcnt < dp->sh->mpm_proto_other_ctx->pattern_cnt) - de_ctx->mpm_max_patcnt = dp->sh->mpm_proto_other_ctx->pattern_cnt; - - de_ctx->mpm_tot_patcnt += dp->sh->mpm_proto_other_ctx->pattern_cnt; - } - if (dp->sh->mpm_uri_ctx_ts != NULL) { - if (de_ctx->mpm_uri_max_patcnt < dp->sh->mpm_uri_ctx_ts->pattern_cnt) - de_ctx->mpm_uri_max_patcnt = dp->sh->mpm_uri_ctx_ts->pattern_cnt; - - de_ctx->mpm_uri_tot_patcnt += dp->sh->mpm_uri_ctx_ts->pattern_cnt; - } - /* dbg */ - if (!(dp->sh->flags & SIG_GROUP_HEAD_MPM_COPY) && dp->sh->mpm_proto_tcp_ctx_ts) { - de_ctx->mpm_memory_size += dp->sh->mpm_proto_tcp_ctx_ts->memory_size; - } - if (!(dp->sh->flags & SIG_GROUP_HEAD_MPM_COPY) && dp->sh->mpm_proto_tcp_ctx_tc) { - de_ctx->mpm_memory_size += dp->sh->mpm_proto_tcp_ctx_tc->memory_size; - } - if (!(dp->sh->flags & SIG_GROUP_HEAD_MPM_COPY) && dp->sh->mpm_proto_udp_ctx_ts) { - de_ctx->mpm_memory_size += dp->sh->mpm_proto_udp_ctx_ts->memory_size; - } - if (!(dp->sh->flags & SIG_GROUP_HEAD_MPM_COPY) && dp->sh->mpm_proto_udp_ctx_tc) { - de_ctx->mpm_memory_size += dp->sh->mpm_proto_udp_ctx_tc->memory_size; - } - if (!(dp->sh->flags & SIG_GROUP_HEAD_MPM_COPY) && dp->sh->mpm_proto_other_ctx) { - de_ctx->mpm_memory_size += dp->sh->mpm_proto_other_ctx->memory_size; - } - if (!(dp->sh->flags & SIG_GROUP_HEAD_MPM_URI_COPY) && dp->sh->mpm_uri_ctx_ts) { - de_ctx->mpm_memory_size += dp->sh->mpm_uri_ctx_ts->memory_size; - } - SigGroupHeadDPortHashAdd(de_ctx, dp->sh); SigGroupHeadStore(de_ctx, dp->sh); de_ctx->gh_unique++; @@ -4250,20 +4141,8 @@ int SigAddressPrepareStage3(DetectEngineCtx *de_ctx) DetectPortSpHashFree(de_ctx); if (!(de_ctx->flags & DE_QUIET)) { - SCLogDebug("MPM memory %" PRIuMAX " (dynamic %" PRIu32 ", ctxs %" PRIuMAX ", avg per ctx %" PRIu32 ")", - de_ctx->mpm_memory_size + ((de_ctx->mpm_unique + de_ctx->mpm_uri_unique) * (uintmax_t)sizeof(MpmCtx)), - de_ctx->mpm_memory_size, ((de_ctx->mpm_unique + de_ctx->mpm_uri_unique) * (uintmax_t)sizeof(MpmCtx)), - de_ctx->mpm_unique ? de_ctx->mpm_memory_size / de_ctx->mpm_unique: 0); - SCLogDebug("max sig id %" PRIu32 ", array size %" PRIu32 "", DetectEngineGetMaxSigId(de_ctx), DetectEngineGetMaxSigId(de_ctx) / 8 + 1); SCLogDebug("signature group heads: unique %" PRIu32 ", copies %" PRIu32 ".", de_ctx->gh_unique, de_ctx->gh_reuse); - SCLogDebug("MPM instances: %" PRIu32 " unique, copies %" PRIu32 " (none %" PRIu32 ").", - de_ctx->mpm_unique, de_ctx->mpm_reuse, de_ctx->mpm_none); - SCLogDebug("MPM (URI) instances: %" PRIu32 " unique, copies %" PRIu32 " (none %" PRIu32 ").", - de_ctx->mpm_uri_unique, de_ctx->mpm_uri_reuse, de_ctx->mpm_uri_none); - SCLogDebug("MPM max patcnt %" PRIu32 ", avg %" PRIu32 "", de_ctx->mpm_max_patcnt, de_ctx->mpm_unique?de_ctx->mpm_tot_patcnt/de_ctx->mpm_unique:0); - if (de_ctx->mpm_uri_tot_patcnt && de_ctx->mpm_uri_unique) - SCLogDebug("MPM (URI) max patcnt %" PRIu32 ", avg %" PRIu32 " (%" PRIu32 "/%" PRIu32 ")", de_ctx->mpm_uri_max_patcnt, de_ctx->mpm_uri_tot_patcnt/de_ctx->mpm_uri_unique, de_ctx->mpm_uri_tot_patcnt, de_ctx->mpm_uri_unique); SCLogDebug("port maxgroups: %" PRIu32 ", avg %" PRIu32 ", tot %" PRIu32 "", g_groupportlist_maxgroups, g_groupportlist_groupscnt ? g_groupportlist_totgroups/g_groupportlist_groupscnt : 0, g_groupportlist_totgroups); SCLogInfo("building signature grouping structure, stage 3: building destination address lists... complete"); @@ -4376,6 +4255,9 @@ int SigAddressPrepareStage4(DetectEngineCtx *de_ctx) SCLogDebug("filestore count %u", sgh->filestore_cnt); SigGroupHeadBuildNonMpmArray(de_ctx, sgh); + + sgh->mpm_uricontent_minlen = SigGroupHeadGetMinMpmSize(de_ctx, sgh, DETECT_SM_LIST_UMATCH); + SCLogDebug("http_uri content min mpm len: %u", sgh->mpm_uricontent_minlen); } if (de_ctx->decoder_event_sgh != NULL) { @@ -4471,7 +4353,7 @@ int SigAddressPrepareStage5(DetectEngineCtx *de_ctx) DetectPort *dp = sp->dst_ph; for ( ; dp != NULL; dp = dp->next) { printf(" 4 Dst port(range): "); DetectPortPrint(dp); - printf(" (sigs %" PRIu32 ", sgh %p, maxlen %" PRIu32 ")", dp->sh->sig_cnt, dp->sh, dp->sh->mpm_content_maxlen); + printf(" (sigs %" PRIu32 ", sgh %p, minlen %" PRIu32 ")", dp->sh->sig_cnt, dp->sh, dp->sh->mpm_content_minlen); #ifdef PRINTSIGS printf(" - "); for (u = 0; u < dp->sh->sig_cnt; u++) { @@ -9913,10 +9795,6 @@ static int SigTestSgh01 (void) printf("internal id != 0: "); goto end; } - if (de_ctx->sig_list->mpm_content_maxlen != 3) { - printf("de_ctx->sig_list->mpm_content_maxlen %u, expected 3: ", de_ctx->sig_list->mpm_content_maxlen); - goto end; - } de_ctx->sig_list->next = SigInit(de_ctx,"alert tcp any any -> any 81 (msg:\"2\"; content:\"two\"; content:\"abcd\"; sid:2;)"); if (de_ctx->sig_list->next == NULL) { @@ -9927,10 +9805,6 @@ static int SigTestSgh01 (void) printf("internal id != 1: "); goto end; } - if (de_ctx->sig_list->next->mpm_content_maxlen != 4) { - printf("de_ctx->sig_list->mpm_content_maxlen %u, expected 4: ", de_ctx->sig_list->next->mpm_content_maxlen); - goto end; - } de_ctx->sig_list->next->next = SigInit(de_ctx,"alert tcp any any -> any 80 (msg:\"3\"; content:\"three\"; sid:3;)"); if (de_ctx->sig_list->next->next == NULL) { @@ -9941,10 +9815,6 @@ static int SigTestSgh01 (void) printf("internal id != 2: "); goto end; } - if (de_ctx->sig_list->next->next->mpm_content_maxlen != 5) { - printf("de_ctx->sig_list->next->next->mpm_content_maxlen %u, expected 5: ", de_ctx->sig_list->next->next->mpm_content_maxlen); - goto end; - } SigGroupBuild(de_ctx); DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx); @@ -9961,8 +9831,8 @@ static int SigTestSgh01 (void) printf("sgh->sig_cnt %u\n", sgh->sig_cnt); printf("sgh->sig_size %u\n", sgh->sig_size); #endif - if (sgh->mpm_content_maxlen != 3) { - printf("sgh->mpm_content_maxlen %u, expected 3: ", sgh->mpm_content_maxlen); + if (sgh->mpm_content_minlen != 3) { + printf("sgh->mpm_content_minlen %u, expected 3: ", sgh->mpm_content_minlen); goto end; } @@ -10006,13 +9876,13 @@ static int SigTestSgh01 (void) #if 0 printf("-\n"); - printf("sgh2->mpm_content_maxlen %u\n", sgh2->mpm_content_maxlen); - printf("sgh2->mpm_uricontent_maxlen %u\n", sgh2->mpm_uricontent_maxlen); + printf("sgh2->mpm_content_minlen %u\n", sgh2->mpm_content_minlen); + printf("sgh2->mpm_uricontent_minlen %u\n", sgh2->mpm_uricontent_minlen); printf("sgh2->sig_cnt %u\n", sgh2->sig_cnt); printf("sgh2->sig_size %u\n", sgh2->sig_size); #endif - if (sgh2->mpm_content_maxlen != 4) { - printf("sgh2->mpm_content_maxlen %u, expected 4: ", sgh2->mpm_content_maxlen); + if (sgh2->mpm_content_minlen != 4) { + printf("sgh2->mpm_content_minlen %u, expected 4: ", sgh2->mpm_content_minlen); goto end; } @@ -10084,8 +9954,8 @@ static int SigTestSgh02 (void) goto end; } - if (sgh->mpm_content_maxlen != 3) { - printf("sgh->mpm_content_maxlen %u, expected 3: ", sgh->mpm_content_maxlen); + if (sgh->mpm_content_minlen != 3) { + printf("sgh->mpm_content_minlen %u, expected 3: ", sgh->mpm_content_minlen); goto end; } @@ -10108,8 +9978,8 @@ static int SigTestSgh02 (void) goto end; } #if 0 - printf("sgh->mpm_content_maxlen %u\n", sgh->mpm_content_maxlen); - printf("sgh->mpm_uricontent_maxlen %u\n", sgh->mpm_uricontent_maxlen); + printf("sgh->mpm_content_minlen %u\n", sgh->mpm_content_minlen); + printf("sgh->mpm_uricontent_minlen %u\n", sgh->mpm_uricontent_minlen); printf("sgh->sig_cnt %u\n", sgh->sig_cnt); printf("sgh->sig_size %u\n", sgh->sig_size); printf("sgh->refcnt %u\n", sgh->refcnt); @@ -10122,8 +9992,8 @@ static int SigTestSgh02 (void) goto end; } - if (sgh->mpm_content_maxlen != 3) { - printf("sgh->mpm_content_maxlen %u, expected 3: ", sgh->mpm_content_maxlen); + if (sgh->mpm_content_minlen != 3) { + printf("sgh->mpm_content_minlen %u, expected 3: ", sgh->mpm_content_minlen); goto end; } @@ -10144,8 +10014,8 @@ static int SigTestSgh02 (void) goto end; } #if 0 - printf("sgh->mpm_content_maxlen %u\n", sgh->mpm_content_maxlen); - printf("sgh->mpm_uricontent_maxlen %u\n", sgh->mpm_uricontent_maxlen); + printf("sgh->mpm_content_minlen %u\n", sgh->mpm_content_minlen); + printf("sgh->mpm_uricontent_minlen %u\n", sgh->mpm_uricontent_minlen); printf("sgh->sig_cnt %u\n", sgh->sig_cnt); printf("sgh->sig_size %u\n", sgh->sig_size); printf("sgh->refcnt %u\n", sgh->refcnt); @@ -10158,8 +10028,8 @@ static int SigTestSgh02 (void) goto end; } - if (sgh->mpm_content_maxlen != 3) { - printf("sgh->mpm_content_maxlen %u, expected 3: ", sgh->mpm_content_maxlen); + if (sgh->mpm_content_minlen != 3) { + printf("sgh->mpm_content_minlen %u, expected 3: ", sgh->mpm_content_minlen); goto end; } @@ -10173,8 +10043,8 @@ static int SigTestSgh02 (void) goto end; } #if 0 - printf("sgh->mpm_content_maxlen %u\n", sgh->mpm_content_maxlen); - printf("sgh->mpm_uricontent_maxlen %u\n", sgh->mpm_uricontent_maxlen); + printf("sgh->mpm_content_minlen %u\n", sgh->mpm_content_minlen); + printf("sgh->mpm_uricontent_minlen %u\n", sgh->mpm_uricontent_minlen); printf("sgh->sig_cnt %u\n", sgh->sig_cnt); printf("sgh->sig_size %u\n", sgh->sig_size); printf("sgh->refcnt %u\n", sgh->refcnt); @@ -10188,8 +10058,8 @@ static int SigTestSgh02 (void) goto end; } - if (sgh->mpm_content_maxlen != 3) { - printf("sgh->mpm_content_maxlen %u, expected 3: ", sgh->mpm_content_maxlen); + if (sgh->mpm_content_minlen != 3) { + printf("sgh->mpm_content_minlen %u, expected 3: ", sgh->mpm_content_minlen); goto end; } @@ -10203,8 +10073,8 @@ static int SigTestSgh02 (void) goto end; } #if 0 - printf("sgh->mpm_content_maxlen %u\n", sgh->mpm_content_maxlen); - printf("sgh->mpm_uricontent_maxlen %u\n", sgh->mpm_uricontent_maxlen); + printf("sgh->mpm_content_minlen %u\n", sgh->mpm_content_minlen); + printf("sgh->mpm_uricontent_minlen %u\n", sgh->mpm_uricontent_minlen); printf("sgh->sig_cnt %u\n", sgh->sig_cnt); printf("sgh->sig_size %u\n", sgh->sig_size); printf("sgh->refcnt %u\n", sgh->refcnt); @@ -10281,14 +10151,14 @@ static int SigTestSgh03 (void) } #if 0 printf("-\n"); - printf("sgh->mpm_content_maxlen %u\n", sgh->mpm_content_maxlen); - printf("sgh->mpm_uricontent_maxlen %u\n", sgh->mpm_uricontent_maxlen); + printf("sgh->mpm_content_minlen %u\n", sgh->mpm_content_minlen); + printf("sgh->mpm_uricontent_minlen %u\n", sgh->mpm_uricontent_minlen); printf("sgh->sig_cnt %u\n", sgh->sig_cnt); printf("sgh->sig_size %u\n", sgh->sig_size); printf("sgh->refcnt %u\n", sgh->refcnt); #endif - if (sgh->mpm_content_maxlen != 3) { - printf("sgh->mpm_content_maxlen %u, expected 3: ", sgh->mpm_content_maxlen); + if (sgh->mpm_content_minlen != 3) { + printf("sgh->mpm_content_minlen %u, expected 3: ", sgh->mpm_content_minlen); goto end; } @@ -10321,8 +10191,8 @@ static int SigTestSgh03 (void) #if 0 printf("-\n"); printf("sgh %p\n", sgh); - printf("sgh->mpm_content_maxlen %u\n", sgh->mpm_content_maxlen); - printf("sgh->mpm_uricontent_maxlen %u\n", sgh->mpm_uricontent_maxlen); + printf("sgh->mpm_content_minlen %u\n", sgh->mpm_content_minlen); + printf("sgh->mpm_uricontent_minlen %u\n", sgh->mpm_uricontent_minlen); printf("sgh->sig_cnt %u\n", sgh->sig_cnt); printf("sgh->sig_size %u\n", sgh->sig_size); printf("sgh->refcnt %u\n", sgh->refcnt); @@ -10344,8 +10214,8 @@ static int SigTestSgh03 (void) goto end; } - if (sgh->mpm_content_maxlen != 3) { - printf("sgh->mpm_content_maxlen %u, expected 3 (%x): ", sgh->mpm_content_maxlen, p->dst.addr_data32[0]); + if (sgh->mpm_content_minlen != 3) { + printf("sgh->mpm_content_minlen %u, expected 3 (%x): ", sgh->mpm_content_minlen, p->dst.addr_data32[0]); goto end; } @@ -10359,14 +10229,14 @@ static int SigTestSgh03 (void) } #if 0 printf("-\n"); - printf("sgh->mpm_content_maxlen %u\n", sgh->mpm_content_maxlen); - printf("sgh->mpm_uricontent_maxlen %u\n", sgh->mpm_uricontent_maxlen); + printf("sgh->mpm_content_minlen %u\n", sgh->mpm_content_minlen); + printf("sgh->mpm_uricontent_minlen %u\n", sgh->mpm_uricontent_minlen); printf("sgh->sig_cnt %u\n", sgh->sig_cnt); printf("sgh->sig_size %u\n", sgh->sig_size); printf("sgh->refcnt %u\n", sgh->refcnt); #endif - if (sgh->mpm_content_maxlen != 3) { - printf("sgh->mpm_content_maxlen %u, expected 3: ", sgh->mpm_content_maxlen); + if (sgh->mpm_content_minlen != 3) { + printf("sgh->mpm_content_minlen %u, expected 3: ", sgh->mpm_content_minlen); goto end; } @@ -10452,8 +10322,8 @@ static int SigTestSgh04 (void) goto end; } - if (sgh->mpm_content_maxlen != 3) { - printf("sgh->mpm_content_maxlen %u, expected 3: ", sgh->mpm_content_maxlen); + if (sgh->mpm_content_minlen != 3) { + printf("sgh->mpm_content_minlen %u, expected 3: ", sgh->mpm_content_minlen); goto end; } @@ -10476,8 +10346,8 @@ static int SigTestSgh04 (void) goto end; } #if 0 - printf("sgh->mpm_content_maxlen %u\n", sgh->mpm_content_maxlen); - printf("sgh->mpm_uricontent_maxlen %u\n", sgh->mpm_uricontent_maxlen); + printf("sgh->mpm_content_minlen %u\n", sgh->mpm_content_minlen); + printf("sgh->mpm_uricontent_minlen %u\n", sgh->mpm_uricontent_minlen); printf("sgh->sig_cnt %u\n", sgh->sig_cnt); printf("sgh->sig_size %u\n", sgh->sig_size); printf("sgh->refcnt %u\n", sgh->refcnt); @@ -10490,8 +10360,8 @@ static int SigTestSgh04 (void) goto end; } - if (sgh->mpm_content_maxlen != 3) { - printf("sgh->mpm_content_maxlen %u, expected 3: ", sgh->mpm_content_maxlen); + if (sgh->mpm_content_minlen != 3) { + printf("sgh->mpm_content_minlen %u, expected 3: ", sgh->mpm_content_minlen); goto end; } @@ -10512,8 +10382,8 @@ static int SigTestSgh04 (void) goto end; } #if 0 - printf("sgh->mpm_content_maxlen %u\n", sgh->mpm_content_maxlen); - printf("sgh->mpm_uricontent_maxlen %u\n", sgh->mpm_uricontent_maxlen); + printf("sgh->mpm_content_minlen %u\n", sgh->mpm_content_minlen); + printf("sgh->mpm_uricontent_minlen %u\n", sgh->mpm_uricontent_minlen); printf("sgh->sig_cnt %u\n", sgh->sig_cnt); printf("sgh->sig_size %u\n", sgh->sig_size); printf("sgh->refcnt %u\n", sgh->refcnt); @@ -10526,8 +10396,8 @@ static int SigTestSgh04 (void) goto end; } - if (sgh->mpm_content_maxlen != 3) { - printf("sgh->mpm_content_maxlen %u, expected 3: ", sgh->mpm_content_maxlen); + if (sgh->mpm_content_minlen != 3) { + printf("sgh->mpm_content_minlen %u, expected 3: ", sgh->mpm_content_minlen); goto end; } @@ -10541,8 +10411,8 @@ static int SigTestSgh04 (void) goto end; } #if 0 - printf("sgh->mpm_content_maxlen %u\n", sgh->mpm_content_maxlen); - printf("sgh->mpm_uricontent_maxlen %u\n", sgh->mpm_uricontent_maxlen); + printf("sgh->mpm_content_minlen %u\n", sgh->mpm_content_minlen); + printf("sgh->mpm_uricontent_minlen %u\n", sgh->mpm_uricontent_minlen); printf("sgh->sig_cnt %u\n", sgh->sig_cnt); printf("sgh->sig_size %u\n", sgh->sig_size); printf("sgh->refcnt %u\n", sgh->refcnt); @@ -10556,14 +10426,14 @@ static int SigTestSgh04 (void) } #if 0 printf("-\n"); - printf("sgh->mpm_content_maxlen %u\n", sgh->mpm_content_maxlen); - printf("sgh->mpm_uricontent_maxlen %u\n", sgh->mpm_uricontent_maxlen); + printf("sgh->mpm_content_minlen %u\n", sgh->mpm_content_minlen); + printf("sgh->mpm_uricontent_minlen %u\n", sgh->mpm_uricontent_minlen); printf("sgh->sig_cnt %u\n", sgh->sig_cnt); printf("sgh->sig_size %u\n", sgh->sig_size); printf("sgh->refcnt %u\n", sgh->refcnt); #endif - if (sgh->mpm_content_maxlen != 3) { - printf("sgh->mpm_content_maxlen %u, expected 3: ", sgh->mpm_content_maxlen); + if (sgh->mpm_content_minlen != 3) { + printf("sgh->mpm_content_minlen %u, expected 3: ", sgh->mpm_content_minlen); goto end; } diff --git a/framework/src/suricata/src/detect.h b/framework/src/suricata/src/detect.h index 30adc9c4..8d131b92 100644 --- a/framework/src/suricata/src/detect.h +++ b/framework/src/suricata/src/detect.h @@ -449,10 +449,8 @@ typedef struct Signature_ { SigMatch *dsize_sm; /* the fast pattern added from this signature */ SigMatch *mpm_sm; - /* helper for init phase */ - uint16_t mpm_content_maxlen; - uint16_t mpm_uricontent_maxlen; + /* SigMatch list used for adding content and friends. E.g. file_data; */ int list; /* Be careful, this pointer is only valid while parsing the sig, @@ -595,13 +593,8 @@ typedef struct DetectEngineCtx_ { /* main sigs */ DetectEngineLookupFlow flow_gh[FLOW_STATES]; - uint32_t mpm_unique, mpm_reuse, mpm_none, - mpm_uri_unique, mpm_uri_reuse, mpm_uri_none; uint32_t gh_unique, gh_reuse; - uint32_t mpm_max_patcnt, mpm_min_patcnt, mpm_tot_patcnt, - mpm_uri_max_patcnt, mpm_uri_min_patcnt, mpm_uri_tot_patcnt; - /* init phase vars */ HashListTable *sgh_hash_table; @@ -622,9 +615,6 @@ typedef struct DetectEngineCtx_ { /* hash table used to cull out duplicate sigs */ HashListTable *dup_sig_hash_table; - /* memory counters */ - uint32_t mpm_memory_size; - DetectEngineIPOnlyCtx io_ctx; ThresholdCtx ths_ctx; @@ -994,7 +984,8 @@ typedef struct SigGroupHead_ { /* number of sigs in this head */ SigIntId sig_cnt; - uint16_t mpm_content_maxlen; + /* track min pattern length for content. Used in grouping */ + uint16_t mpm_content_minlen; /** array of masks, used to check multiple masks against * a packet using SIMD. */ @@ -1034,7 +1025,7 @@ typedef struct SigGroupHead_ { MpmCtx *mpm_hsmd_ctx_tc; MpmCtx *mpm_hscd_ctx_tc; - uint16_t mpm_uricontent_maxlen; + uint16_t mpm_uricontent_minlen; /**< len of shortest mpm pattern in sgh */ /** the number of signatures in this sgh that have the filestore keyword * set. */ diff --git a/framework/src/suricata/src/flow-hash.c b/framework/src/suricata/src/flow-hash.c index 7a151199..9ddb3713 100644 --- a/framework/src/suricata/src/flow-hash.c +++ b/framework/src/suricata/src/flow-hash.c @@ -35,6 +35,7 @@ #include "flow-util.h" #include "flow-private.h" #include "flow-manager.h" +#include "flow-storage.h" #include "app-layer-parser.h" #include "util-time.h" @@ -472,7 +473,7 @@ static Flow *FlowGetNew(ThreadVars *tv, DecodeThreadVars *dtv, const Packet *p) f = FlowDequeue(&flow_spare_q); if (f == NULL) { /* If we reached the max memcap, we get a used flow */ - if (!(FLOW_CHECK_MEMCAP(sizeof(Flow)))) { + if (!(FLOW_CHECK_MEMCAP(sizeof(Flow) + FlowStorageSize()))) { /* declare state of emergency */ if (!(SC_ATOMIC_GET(flow_flags) & FLOW_EMERGENCY)) { SC_ATOMIC_OR(flow_flags, FLOW_EMERGENCY); @@ -485,6 +486,11 @@ static Flow *FlowGetNew(ThreadVars *tv, DecodeThreadVars *dtv, const Packet *p) f = FlowGetUsedFlow(tv, dtv); if (f == NULL) { + /* max memcap reached, so increments the counter */ + if (tv != NULL && dtv != NULL) { + StatsIncr(tv, dtv->counter_flow_memcap); + } + /* very rare, but we can fail. Just giving up */ return NULL; } @@ -494,6 +500,9 @@ static Flow *FlowGetNew(ThreadVars *tv, DecodeThreadVars *dtv, const Packet *p) /* now see if we can alloc a new flow */ f = FlowAlloc(); if (f == NULL) { + if (tv != NULL && dtv != NULL) { + StatsIncr(tv, dtv->counter_flow_memcap); + } return NULL; } diff --git a/framework/src/suricata/src/flow.c b/framework/src/suricata/src/flow.c index 42099c3a..abd9e7e5 100644 --- a/framework/src/suricata/src/flow.c +++ b/framework/src/suricata/src/flow.c @@ -443,7 +443,7 @@ void FlowInitConfig(char quiet) /* pre allocate flows */ for (i = 0; i < flow_config.prealloc; i++) { - if (!(FLOW_CHECK_MEMCAP(sizeof(Flow)))) { + if (!(FLOW_CHECK_MEMCAP(sizeof(Flow) + FlowStorageSize()))) { SCLogError(SC_ERR_FLOW_INIT, "preallocating flows failed: " "max flow memcap reached. Memcap %"PRIu64", " "Memuse %"PRIu64".", flow_config.memcap, @@ -462,7 +462,7 @@ void FlowInitConfig(char quiet) if (quiet == FALSE) { SCLogInfo("preallocated %" PRIu32 " flows of size %" PRIuMAX "", - flow_spare_q.len, (uintmax_t)sizeof(Flow)); + flow_spare_q.len, (uintmax_t)(sizeof(Flow) + + FlowStorageSize())); SCLogInfo("flow memory usage: %llu bytes, maximum: %"PRIu64, SC_ATOMIC_GET(flow_memuse), flow_config.memcap); } diff --git a/framework/src/suricata/src/host-storage.c b/framework/src/suricata/src/host-storage.c index 6748089c..fe157692 100644 --- a/framework/src/suricata/src/host-storage.c +++ b/framework/src/suricata/src/host-storage.c @@ -32,16 +32,66 @@ unsigned int HostStorageSize(void) return StorageGetSize(STORAGE_HOST); } -void *HostGetStorageById(Host *h, int id) -{ - return StorageGetById((Storage *)((void *)h + sizeof(Host)), STORAGE_HOST, id); +/** \defgroup hoststorage Host storage API + * + * The Host storage API is a per-host storage. It is a mean to extend + * the Host structure with arbitrary data. + * + * You have first to register the storage via HostStorageRegister() during + * the init of your module. Then you can attach data via HostSetStorageById() + * and access them via HostGetStorageById(). + * @{ + */ + +/** + * \brief Register a Host storage + * + * \param name the name of the storage + * \param size integer coding the size of the stored value (sizeof(void *) is best choice here) + * \param Alloc allocation function for the storage (can be null) + * \param Free free function for the new storage + * + * \retval The ID of the newly register storage that will be used to access data + * + * It has to be called once during the init of the sub system + */ + +int HostStorageRegister(const char *name, const unsigned int size, void *(*Alloc)(unsigned int), void (*Free)(void *)) { + return StorageRegister(STORAGE_HOST, name, size, Alloc, Free); } +/** + * \brief Store a pointer in a given Host storage + * + * \param h a pointer to the Host + * \param id the id of the storage (return of HostStorageRegister() call) + * \param ptr pointer to the data to store + */ + int HostSetStorageById(Host *h, int id, void *ptr) { return StorageSetById((Storage *)((void *)h + sizeof(Host)), STORAGE_HOST, id, ptr); } +/** + * \brief Get a value from a given Host storage + * + * \param h a pointer to the Host + * \param id the id of the storage (return of HostStorageRegister() call) + * + */ + +void *HostGetStorageById(Host *h, int id) +{ + return StorageGetById((Storage *)((void *)h + sizeof(Host)), STORAGE_HOST, id); +} + +/** + * @} + */ + +/* Start of "private" function */ + void *HostAllocStorageById(Host *h, int id) { return StorageAllocByIdPrealloc((Storage *)((void *)h + sizeof(Host)), STORAGE_HOST, id); @@ -58,9 +108,6 @@ void HostFreeStorage(Host *h) StorageFreeAll((Storage *)((void *)h + sizeof(Host)), STORAGE_HOST); } -int HostStorageRegister(const char *name, const unsigned int size, void *(*Alloc)(unsigned int), void (*Free)(void *)) { - return StorageRegister(STORAGE_HOST, name, size, Alloc, Free); -} #ifdef UNITTESTS diff --git a/framework/src/suricata/src/host.c b/framework/src/suricata/src/host.c index 7c3c5841..a28639cc 100644 --- a/framework/src/suricata/src/host.c +++ b/framework/src/suricata/src/host.c @@ -48,6 +48,10 @@ static Host *HostGetUsedHost(void); /** queue with spare hosts */ static HostQueue host_spare_q; +/** size of the host object. Maybe updated in HostInitConfig to include + * the storage APIs additions. */ +static uint16_t g_host_size = sizeof(Host); + uint32_t HostSpareQueueGetSize(void) { return HostQueueLen(&host_spare_q); @@ -61,19 +65,16 @@ void HostMoveToSpare(Host *h) Host *HostAlloc(void) { - size_t size = sizeof(Host) + HostStorageSize(); - - if (!(HOST_CHECK_MEMCAP(size))) { + if (!(HOST_CHECK_MEMCAP(g_host_size))) { return NULL; } + (void) SC_ATOMIC_ADD(host_memuse, g_host_size); - (void) SC_ATOMIC_ADD(host_memuse, size); - - Host *h = SCMalloc(size); + Host *h = SCMalloc(g_host_size); if (unlikely(h == NULL)) goto error; - memset(h, 0x00, size); + memset(h, 0x00, g_host_size); SCMutexInit(&h->m, NULL); SC_ATOMIC_INIT(h->use_cnt); @@ -91,7 +92,7 @@ void HostFree(Host *h) SC_ATOMIC_DESTROY(h->use_cnt); SCMutexDestroy(&h->m); SCFree(h); - (void) SC_ATOMIC_SUB(host_memuse, (sizeof(Host) + HostStorageSize())); + (void) SC_ATOMIC_SUB(host_memuse, g_host_size); } } @@ -130,6 +131,8 @@ void HostClearMemory(Host *h) void HostInitConfig(char quiet) { SCLogDebug("initializing host engine..."); + if (HostStorageSize() > 0) + g_host_size = sizeof(Host) + HostStorageSize(); memset(&host_config, 0, sizeof(host_config)); //SC_ATOMIC_INIT(flow_flags); @@ -214,11 +217,11 @@ void HostInitConfig(char quiet) /* pre allocate hosts */ for (i = 0; i < host_config.prealloc; i++) { - if (!(HOST_CHECK_MEMCAP(sizeof(Host)))) { + if (!(HOST_CHECK_MEMCAP(g_host_size))) { SCLogError(SC_ERR_HOST_INIT, "preallocating hosts failed: " "max host memcap reached. Memcap %"PRIu64", " "Memuse %"PRIu64".", host_config.memcap, - ((uint64_t)SC_ATOMIC_GET(host_memuse) + (uint64_t)sizeof(Host))); + ((uint64_t)SC_ATOMIC_GET(host_memuse) + g_host_size)); exit(EXIT_FAILURE); } @@ -231,8 +234,8 @@ void HostInitConfig(char quiet) } if (quiet == FALSE) { - SCLogInfo("preallocated %" PRIu32 " hosts of size %" PRIuMAX "", - host_spare_q.len, (uintmax_t)sizeof(Host)); + SCLogInfo("preallocated %" PRIu32 " hosts of size %" PRIu16 "", + host_spare_q.len, g_host_size); SCLogInfo("host memory usage: %llu bytes, maximum: %"PRIu64, SC_ATOMIC_GET(host_memuse), host_config.memcap); } @@ -386,7 +389,7 @@ static Host *HostGetNew(Address *a) h = HostDequeue(&host_spare_q); if (h == NULL) { /* If we reached the max memcap, we get a used host */ - if (!(HOST_CHECK_MEMCAP(sizeof(Host)))) { + if (!(HOST_CHECK_MEMCAP(g_host_size))) { /* declare state of emergency */ //if (!(SC_ATOMIC_GET(host_flags) & HOST_EMERGENCY)) { // SC_ATOMIC_OR(host_flags, HOST_EMERGENCY); diff --git a/framework/src/suricata/src/ippair.c b/framework/src/suricata/src/ippair.c index 780ee6ce..79ecb76c 100644 --- a/framework/src/suricata/src/ippair.c +++ b/framework/src/suricata/src/ippair.c @@ -47,6 +47,10 @@ static IPPair *IPPairGetUsedIPPair(void); /** queue with spare ippairs */ static IPPairQueue ippair_spare_q; +/** size of the ippair object. Maybe updated in IPPairInitConfig to include + * the storage APIs additions. */ +static uint16_t g_ippair_size = sizeof(IPPair); + uint32_t IPPairSpareQueueGetSize(void) { return IPPairQueueLen(&ippair_spare_q); @@ -60,19 +64,17 @@ void IPPairMoveToSpare(IPPair *h) IPPair *IPPairAlloc(void) { - size_t size = sizeof(IPPair) + IPPairStorageSize(); - - if (!(IPPAIR_CHECK_MEMCAP(size))) { + if (!(IPPAIR_CHECK_MEMCAP(g_ippair_size))) { return NULL; } - (void) SC_ATOMIC_ADD(ippair_memuse, size); + (void) SC_ATOMIC_ADD(ippair_memuse, g_ippair_size); - IPPair *h = SCMalloc(size); + IPPair *h = SCMalloc(g_ippair_size); if (unlikely(h == NULL)) goto error; - memset(h, 0x00, size); + memset(h, 0x00, g_ippair_size); SCMutexInit(&h->m, NULL); SC_ATOMIC_INIT(h->use_cnt); @@ -90,7 +92,7 @@ void IPPairFree(IPPair *h) SC_ATOMIC_DESTROY(h->use_cnt); SCMutexDestroy(&h->m); SCFree(h); - (void) SC_ATOMIC_SUB(ippair_memuse, (sizeof(IPPair) + IPPairStorageSize())); + (void) SC_ATOMIC_SUB(ippair_memuse, g_ippair_size); } } @@ -125,6 +127,8 @@ void IPPairClearMemory(IPPair *h) void IPPairInitConfig(char quiet) { SCLogDebug("initializing ippair engine..."); + if (IPPairStorageSize() > 0) + g_ippair_size = sizeof(IPPair) + IPPairStorageSize(); memset(&ippair_config, 0, sizeof(ippair_config)); //SC_ATOMIC_INIT(flow_flags); @@ -209,11 +213,11 @@ void IPPairInitConfig(char quiet) /* pre allocate ippairs */ for (i = 0; i < ippair_config.prealloc; i++) { - if (!(IPPAIR_CHECK_MEMCAP(sizeof(IPPair)))) { + if (!(IPPAIR_CHECK_MEMCAP(g_ippair_size))) { SCLogError(SC_ERR_IPPAIR_INIT, "preallocating ippairs failed: " "max ippair memcap reached. Memcap %"PRIu64", " "Memuse %"PRIu64".", ippair_config.memcap, - ((uint64_t)SC_ATOMIC_GET(ippair_memuse) + (uint64_t)sizeof(IPPair))); + ((uint64_t)SC_ATOMIC_GET(ippair_memuse) + g_ippair_size)); exit(EXIT_FAILURE); } @@ -226,8 +230,8 @@ void IPPairInitConfig(char quiet) } if (quiet == FALSE) { - SCLogInfo("preallocated %" PRIu32 " ippairs of size %" PRIuMAX "", - ippair_spare_q.len, (uintmax_t)sizeof(IPPair)); + SCLogInfo("preallocated %" PRIu32 " ippairs of size %" PRIu16 "", + ippair_spare_q.len, g_ippair_size); SCLogInfo("ippair memory usage: %llu bytes, maximum: %"PRIu64, SC_ATOMIC_GET(ippair_memuse), ippair_config.memcap); } @@ -382,7 +386,7 @@ static IPPair *IPPairGetNew(Address *a, Address *b) h = IPPairDequeue(&ippair_spare_q); if (h == NULL) { /* If we reached the max memcap, we get a used ippair */ - if (!(IPPAIR_CHECK_MEMCAP(sizeof(IPPair)))) { + if (!(IPPAIR_CHECK_MEMCAP(g_ippair_size))) { /* declare state of emergency */ //if (!(SC_ATOMIC_GET(ippair_flags) & IPPAIR_EMERGENCY)) { // SC_ATOMIC_OR(ippair_flags, IPPAIR_EMERGENCY); diff --git a/framework/src/suricata/src/output-json-alert.c b/framework/src/suricata/src/output-json-alert.c index 3c4219b4..2c0d0171 100644 --- a/framework/src/suricata/src/output-json-alert.c +++ b/framework/src/suricata/src/output-json-alert.c @@ -48,12 +48,15 @@ #include "app-layer-htp-xff.h" #include "util-classification-config.h" #include "util-syslog.h" +#include "util-logopenfile.h" #include "output.h" #include "output-json.h" #include "output-json-http.h" #include "output-json-tls.h" #include "output-json-ssh.h" +#include "output-json-smtp.h" +#include "output-json-email-common.h" #include "util-byte.h" #include "util-privs.h" @@ -61,7 +64,6 @@ #include "util-proto-name.h" #include "util-optimize.h" #include "util-buffer.h" -#include "util-logopenfile.h" #include "util-crypt.h" #define MODULE_NAME "JsonAlertLog" @@ -74,6 +76,7 @@ #define LOG_JSON_HTTP 8 #define LOG_JSON_TLS 16 #define LOG_JSON_SSH 32 +#define LOG_JSON_SMTP 64 #define JSON_STREAM_BUFFER_SIZE 4096 @@ -101,31 +104,6 @@ static int AlertJsonDumpStreamSegmentCallback(const Packet *p, void *data, uint8 return 1; } -/** Handle the case where no JSON support is compiled in. - * - */ -static void AlertJsonHttp(const Flow *f, json_t *js) -{ - HtpState *htp_state = (HtpState *)FlowGetAppState(f); - if (htp_state) { - uint64_t tx_id = AppLayerParserGetTransactionLogId(f->alparser); - htp_tx_t *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP, htp_state, tx_id); - - if (tx) { - json_t *hjs = json_object(); - if (unlikely(hjs == NULL)) - return; - - JsonHttpLogJSONBasic(hjs, tx); - JsonHttpLogJSONExtended(hjs, tx); - - json_object_set_new(js, "http", hjs); - } - } - - return; -} - static void AlertJsonTls(const Flow *f, json_t *js) { SSLState *ssl_state = (SSLState *)FlowGetAppState(f); @@ -168,6 +146,11 @@ void AlertJsonHeader(const Packet *p, const PacketAlert *pa, json_t *js) action = "blocked"; } + /* Add tx_id to root element for correlation with other events. */ + json_object_del(js, "tx_id"); + if (pa->flags & PACKET_ALERT_FLAG_TX) + json_object_set_new(js, "tx_id", json_integer(pa->tx_id)); + json_t *ajs = json_object(); if (ajs == NULL) { json_decref(js); @@ -184,9 +167,6 @@ void AlertJsonHeader(const Packet *p, const PacketAlert *pa, json_t *js) json_string((pa->s->class_msg) ? pa->s->class_msg : "")); json_object_set_new(ajs, "severity", json_integer(pa->s->prio)); - if (pa->flags & PACKET_ALERT_FLAG_TX) - json_object_set_new(ajs, "tx_id", json_integer(pa->tx_id)); - if (p->tenant_id > 0) json_object_set_new(ajs, "tenant_id", json_integer(p->tenant_id)); @@ -198,6 +178,7 @@ static int AlertJson(ThreadVars *tv, JsonAlertLogThread *aft, const Packet *p) { MemBuffer *payload = aft->payload_buffer; AlertJsonOutputCtx *json_output_ctx = aft->json_output_ctx; + json_t *hjs = NULL; int i; @@ -225,8 +206,11 @@ static int AlertJson(ThreadVars *tv, JsonAlertLogThread *aft, const Packet *p) uint16_t proto = FlowGetAppProtocol(p->flow); /* http alert */ - if (proto == ALPROTO_HTTP) - AlertJsonHttp(p->flow, js); + if (proto == ALPROTO_HTTP) { + hjs = JsonHttpAddMetadata(p->flow, pa->tx_id); + if (hjs) + json_object_set_new(js, "http", hjs); + } FLOWLOCK_UNLOCK(p->flow); } @@ -258,6 +242,26 @@ static int AlertJson(ThreadVars *tv, JsonAlertLogThread *aft, const Packet *p) } } + if (json_output_ctx->flags & LOG_JSON_SMTP) { + if (p->flow != NULL) { + FLOWLOCK_RDLOCK(p->flow); + uint16_t proto = FlowGetAppProtocol(p->flow); + + /* http alert */ + if (proto == ALPROTO_SMTP) { + hjs = JsonSMTPAddMetadata(p->flow, pa->tx_id); + if (hjs) + json_object_set_new(js, "smtp", hjs); + + hjs = JsonEmailAddMetadata(p->flow, pa->tx_id); + if (hjs) + json_object_set_new(js, "email", hjs); + } + + FLOWLOCK_UNLOCK(p->flow); + } + } + /* payload */ if (json_output_ctx->flags & (LOG_JSON_PAYLOAD | LOG_JSON_PAYLOAD_BASE64)) { int stream = (p->proto == IPPROTO_TCP) ? @@ -607,6 +611,7 @@ static OutputCtx *JsonAlertLogInitCtxSub(ConfNode *conf, OutputCtx *parent_ctx) const char *http = ConfNodeLookupChildValue(conf, "http"); const char *tls = ConfNodeLookupChildValue(conf, "tls"); const char *ssh = ConfNodeLookupChildValue(conf, "ssh"); + const char *smtp = ConfNodeLookupChildValue(conf, "smtp"); if (ssh != NULL) { if (ConfValIsTrue(ssh)) { @@ -623,6 +628,11 @@ static OutputCtx *JsonAlertLogInitCtxSub(ConfNode *conf, OutputCtx *parent_ctx) json_output_ctx->flags |= LOG_JSON_HTTP; } } + if (smtp != NULL) { + if (ConfValIsTrue(smtp)) { + json_output_ctx->flags |= LOG_JSON_SMTP; + } + } if (payload_printable != NULL) { if (ConfValIsTrue(payload_printable)) { json_output_ctx->flags |= LOG_JSON_PAYLOAD; diff --git a/framework/src/suricata/src/output-json-email-common.c b/framework/src/suricata/src/output-json-email-common.c index 1efa9ce8..88cd3acf 100644 --- a/framework/src/suricata/src/output-json-email-common.c +++ b/framework/src/suricata/src/output-json-email-common.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2014 Open Information Security Foundation +/* Copyright (C) 2007-2015 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -19,6 +19,7 @@ * \file * * \author Tom DeCanio <td@npulsetech.com> + * \author Eric Leblond <eric@regit.org> * * Implements json common email logging portion of the engine. */ @@ -55,137 +56,290 @@ #ifdef HAVE_LIBJANSSON #include <jansson.h> +#define LOG_EMAIL_DEFAULT 0 +#define LOG_EMAIL_EXTENDED (1<<0) +#define LOG_EMAIL_ARRAY (1<<1) /* require array handling */ +#define LOG_EMAIL_COMMA (1<<2) /* require array handling */ +#define LOG_EMAIL_BODY_MD5 (1<<3) +#define LOG_EMAIL_SUBJECT_MD5 (1<<4) + +struct { + char *config_field; + char *email_field; + uint32_t flags; +} email_fields[] = { + { "reply_to", "reply-to", LOG_EMAIL_DEFAULT }, + { "bcc", "bcc", LOG_EMAIL_COMMA|LOG_EMAIL_EXTENDED }, + { "message_id", "message-id", LOG_EMAIL_EXTENDED }, + { "subject", "subject", LOG_EMAIL_EXTENDED }, + { "x_mailer", "x-mailer", LOG_EMAIL_EXTENDED }, + { "user_agent", "user-agent", LOG_EMAIL_EXTENDED }, + { "received", "received", LOG_EMAIL_ARRAY }, + { "x_originating_ip", "x-originating-ip", LOG_EMAIL_DEFAULT }, + { "in_reply_to", "in-reply-to", LOG_EMAIL_DEFAULT }, + { "references", "references", LOG_EMAIL_DEFAULT }, + { "importance", "importance", LOG_EMAIL_DEFAULT }, + { "priority", "priority", LOG_EMAIL_DEFAULT }, + { "sensitivity", "sensitivity", LOG_EMAIL_DEFAULT }, + { "organization", "organization", LOG_EMAIL_DEFAULT }, + { "content_md5", "content-md5", LOG_EMAIL_DEFAULT }, + { "date", "date", LOG_EMAIL_DEFAULT }, + { NULL, NULL, LOG_EMAIL_DEFAULT}, +}; + +static inline char *SkipWhiteSpaceTill(char *p, char *savep) +{ + char *sp = p; + if (unlikely(p == NULL)) { + return NULL; + } + while (((*sp == '\t') || (*sp == ' ')) && (sp < savep)) { + sp++; + } + return sp; +} + +static json_t* JsonEmailJsonArrayFromCommaList(const uint8_t *val, size_t len) +{ + json_t *ajs = json_array(); + if (likely(ajs != NULL)) { + char *savep = NULL; + char *p; + char *sp; + char *to_line = BytesToString((uint8_t *)val, len); + if (likely(to_line != NULL)) { + p = strtok_r(to_line, ",", &savep); + if (p == NULL) { + json_decref(ajs); + SCFree(to_line); + return NULL; + } + sp = SkipWhiteSpaceTill(p, savep); + json_array_append_new(ajs, json_string(sp)); + while ((p = strtok_r(NULL, ",", &savep)) != NULL) { + sp = SkipWhiteSpaceTill(p, savep); + json_array_append_new(ajs, json_string(sp)); + } + } + SCFree(to_line); + } + + return ajs; +} + + +#ifdef HAVE_NSS +static void JsonEmailLogJSONMd5(OutputJsonEmailCtx *email_ctx, json_t *js, SMTPTransaction *tx) +{ + if (email_ctx->flags & LOG_EMAIL_SUBJECT_MD5) { + MimeDecField *field; + MimeDecEntity *entity = tx->msg_tail; + if (entity == NULL) { + return; + } + field = MimeDecFindField(entity, "subject"); + if (field != NULL) { + unsigned char md5[MD5_LENGTH]; + char smd5[256]; + char *value = BytesToString((uint8_t *)field->value , field->value_len); + if (value) { + size_t i,x; + HASH_HashBuf(HASH_AlgMD5, md5, (unsigned char *)value, strlen(value)); + for (i = 0, x = 0; x < sizeof(md5); x++) { + i += snprintf(smd5 + i, 255 - i, "%02x", md5[x]); + } + json_object_set_new(js, "subject_md5", json_string(smd5)); + SCFree(value); + } + } + } + + if (email_ctx->flags & LOG_EMAIL_BODY_MD5) { + MimeDecParseState *mime_state = tx->mime_state; + if (mime_state && mime_state->md5_ctx && (mime_state->state_flag == PARSE_DONE)) { + size_t x; + int i; + char s[256]; + if (likely(s != NULL)) { + for (i = 0, x = 0; x < sizeof(mime_state->md5); x++) { + i += snprintf(s + i, 255-i, "%02x", mime_state->md5[x]); + } + json_object_set_new(js, "body_md5", json_string(s)); + } + } + } +} +#endif + +static int JsonEmailAddToJsonArray(const uint8_t *val, size_t len, void *data) +{ + json_t *ajs = data; + + if (ajs == NULL) + return 0; + char *value = BytesToString((uint8_t *)val, len); + json_array_append_new(ajs, json_string(value)); + SCFree(value); + return 1; +} + +static void JsonEmailLogJSONCustom(OutputJsonEmailCtx *email_ctx, json_t *js, SMTPTransaction *tx) +{ + int f = 0; + MimeDecField *field; + MimeDecEntity *entity = tx->msg_tail; + if (entity == NULL) { + return; + } + + while(email_fields[f].config_field) { + if (((email_ctx->fields & (1ULL<<f)) != 0) + || + ((email_ctx->flags & LOG_EMAIL_EXTENDED) && (email_fields[f].flags & LOG_EMAIL_EXTENDED)) + ) { + if (email_fields[f].flags & LOG_EMAIL_ARRAY) { + json_t *ajs = json_array(); + if (ajs) { + int found = MimeDecFindFieldsForEach(entity, email_fields[f].email_field, JsonEmailAddToJsonArray, ajs); + if (found > 0) { + json_object_set_new(js, email_fields[f].config_field, ajs); + } else { + json_decref(ajs); + } + } + } else if (email_fields[f].flags & LOG_EMAIL_COMMA) { + field = MimeDecFindField(entity, email_fields[f].email_field); + if (field) { + json_t *ajs = JsonEmailJsonArrayFromCommaList(field->value, field->value_len); + if (ajs) { + json_object_set_new(js, email_fields[f].config_field, ajs); + } + } + } else { + field = MimeDecFindField(entity, email_fields[f].email_field); + if (field != NULL) { + char *s = BytesToString((uint8_t *)field->value, + (size_t)field->value_len); + if (likely(s != NULL)) { + json_object_set_new(js, email_fields[f].config_field, json_string(s)); + SCFree(s); + } + } + } + + } + f++; + } +} + /* JSON format logging */ -static TmEcode JsonEmailLogJson(JsonEmailLogThread *aft, json_t *js, const Packet *p, Flow *f, void *state, void *vtx, uint64_t tx_id) +json_t *JsonEmailLogJsonData(const Flow *f, void *state, void *vtx, uint64_t tx_id) { SMTPState *smtp_state; MimeDecParseState *mime_state; MimeDecEntity *entity; - char *protos = NULL; json_t *sjs = json_object(); if (sjs == NULL) { - SCReturnInt(TM_ECODE_FAILED); + SCReturnPtr(NULL, "json_t"); } /* check if we have SMTP state or not */ - AppProto proto = FlowGetAppProtocol(p->flow); + AppProto proto = FlowGetAppProtocol(f); switch (proto) { case ALPROTO_SMTP: smtp_state = (SMTPState *)state; if (smtp_state == NULL) { SCLogDebug("no smtp state, so no request logging"); - SCReturnInt(TM_ECODE_FAILED); + SCReturnPtr(NULL, "json_t"); } SMTPTransaction *tx = vtx; mime_state = tx->mime_state; entity = tx->msg_tail; - protos = "smtp"; SCLogDebug("lets go mime_state %p, entity %p, state_flag %u", mime_state, entity, mime_state ? mime_state->state_flag : 0); break; default: /* don't know how we got here */ - SCReturnInt(TM_ECODE_FAILED); + SCReturnPtr(NULL, "json_t"); } if ((mime_state != NULL)) { if (entity == NULL) { - SCReturnInt(TM_ECODE_FAILED); + SCReturnPtr(NULL, "json_t"); } - if ((entity->header_flags & HDR_IS_LOGGED) == 0) { - MimeDecField *field; - //printf("email LOG\n"); - - /* From: */ - field = MimeDecFindField(entity, "from"); - if (field != NULL) { - char *s = BytesToString((uint8_t *)field->value, - (size_t)field->value_len); - if (likely(s != NULL)) { - //printf("From: \"%s\"\n", s); - json_object_set_new(sjs, "from", json_string(s)); - SCFree(s); - } + json_object_set_new(sjs, "status", + json_string(MimeDecParseStateGetStatus(mime_state))); + + MimeDecField *field; + + /* From: */ + field = MimeDecFindField(entity, "from"); + if (field != NULL) { + char *s = BytesToString((uint8_t *)field->value, + (size_t)field->value_len); + if (likely(s != NULL)) { + //printf("From: \"%s\"\n", s); + char * sp = SkipWhiteSpaceTill(s, s + strlen(s)); + json_object_set_new(sjs, "from", json_string(sp)); + SCFree(s); } + } - /* To: */ - char *to_line = NULL; - field = MimeDecFindField(entity, "to"); - if (field != NULL) { - json_t *js_to = json_array(); - if (likely(js_to != NULL)) { - to_line = BytesToString((uint8_t *)field->value, - (size_t)field->value_len); - if (likely(to_line != NULL)) { - char *savep = NULL; - char *p; - //printf("to_line:: TO: \"%s\" (%d)\n", to_line, strlen(to_line)); - p = strtok_r(to_line, ",", &savep); - //printf("got another addr: \"%s\"\n", p); - json_array_append_new(js_to, json_string(p)); - while ((p = strtok_r(NULL, ",", &savep)) != NULL) { - //printf("got another addr: \"%s\"\n", p); - json_array_append_new(js_to, json_string(&p[strspn(p, " ")])); - } - SCFree(to_line); - } - json_object_set_new(sjs, "to", js_to); - } + /* To: */ + field = MimeDecFindField(entity, "to"); + if (field != NULL) { + json_t *ajs = JsonEmailJsonArrayFromCommaList(field->value, field->value_len); + if (ajs) { + json_object_set_new(sjs, "to", ajs); } + } - /* Cc: */ - char *cc_line = NULL; - field = MimeDecFindField(entity, "cc"); - if (field != NULL) { - json_t *js_cc = json_array(); - if (likely(js_cc != NULL)) { - cc_line = BytesToString((uint8_t *)field->value, - (size_t)field->value_len); - if (likely(cc_line != NULL)) { - char *savep = NULL; - char *p; - //printf("cc_line:: CC: \"%s\" (%d)\n", to_line, strlen(to_line)); - p = strtok_r(cc_line, ",", &savep); - //printf("got another addr: \"%s\"\n", p); - json_array_append_new(js_cc, json_string(p)); - while ((p = strtok_r(NULL, ",", &savep)) != NULL) { - //printf("got another addr: \"%s\"\n", p); - json_array_append_new(js_cc, json_string(&p[strspn(p, " ")])); - } - SCFree(cc_line); - } - json_object_set_new(sjs, "cc", js_cc); - } + /* Cc: */ + field = MimeDecFindField(entity, "cc"); + if (field != NULL) { + json_t *ajs = JsonEmailJsonArrayFromCommaList(field->value, field->value_len); + if (ajs) { + json_object_set_new(sjs, "cc", ajs); } + } - /* Subject: */ - field = MimeDecFindField(entity, "subject"); - if (field != NULL) { - char *s = BytesToString((uint8_t *)field->value, (size_t) field->value_len); - if (likely(s != NULL)) { - //printf("Subject: \"%s\"\n", s); - json_object_set_new(sjs, "subject", json_string(s)); + if (mime_state->stack == NULL || mime_state->stack->top == NULL || mime_state->stack->top->data == NULL) + SCReturnPtr(NULL, "json_t"); + + entity = (MimeDecEntity *)mime_state->stack->top->data; + int attch_cnt = 0; + int url_cnt = 0; + json_t *js_attch = json_array(); + json_t *js_url = json_array(); + if (entity->url_list != NULL) { + MimeDecUrl *url; + for (url = entity->url_list; url != NULL; url = url->next) { + char *s = BytesToString((uint8_t *)url->url, + (size_t)url->url_len); + if (s != NULL) { + json_array_append_new(js_url, + json_string(s)); SCFree(s); + url_cnt += 1; } } + } + for (entity = entity->child; entity != NULL; entity = entity->next) { + if (entity->ctnt_flags & CTNT_IS_ATTACHMENT) { - entity->header_flags |= HDR_IS_LOGGED; - - if (mime_state->stack == NULL || mime_state->stack->top == NULL || mime_state->stack->top->data == NULL) - SCReturnInt(TM_ECODE_OK); - - entity = (MimeDecEntity *)mime_state->stack->top->data; - int attch_cnt = 0; - int url_cnt = 0; - json_t *js_attch = json_array(); - json_t *js_url = json_array(); + char *s = BytesToString((uint8_t *)entity->filename, + (size_t)entity->filename_len); + json_array_append_new(js_attch, + json_string(s)); + SCFree(s); + attch_cnt += 1; + } if (entity->url_list != NULL) { MimeDecUrl *url; for (url = entity->url_list; url != NULL; url = url->next) { char *s = BytesToString((uint8_t *)url->url, (size_t)url->url_len); if (s != NULL) { - //printf("URL: \"%s\"\n", s); json_array_append_new(js_url, json_string(s)); SCFree(s); @@ -193,74 +347,113 @@ static TmEcode JsonEmailLogJson(JsonEmailLogThread *aft, json_t *js, const Packe } } } - for (entity = entity->child; entity != NULL; entity = entity->next) { - if (entity->ctnt_flags & CTNT_IS_ATTACHMENT) { - - char *s = BytesToString((uint8_t *)entity->filename, - (size_t)entity->filename_len); - //printf("found attachment \"%s\"\n", s); - json_array_append_new(js_attch, - json_string(s)); - SCFree(s); - attch_cnt += 1; - } - if (entity->url_list != NULL) { - MimeDecUrl *url; - for (url = entity->url_list; url != NULL; url = url->next) { - char *s = BytesToString((uint8_t *)url->url, - (size_t)url->url_len); - if (s != NULL) { - //printf("URL: \"%s\"\n", s); - json_array_append_new(js_url, - json_string(s)); - SCFree(s); - url_cnt += 1; - } - } - } - } - if (attch_cnt > 0) { - json_object_set_new(sjs, "attachment", js_attch); - } else { - json_decref(js_attch); - } - if (url_cnt > 0) { - json_object_set_new(sjs, "url", js_url); - } else { - json_decref(js_url); - } - json_object_set_new(js, protos, sjs); - -// FLOWLOCK_UNLOCK(p->flow); - SCReturnInt(TM_ECODE_OK); } + if (attch_cnt > 0) { + json_object_set_new(sjs, "attachment", js_attch); + } else { + json_decref(js_attch); + } + if (url_cnt > 0) { + json_object_set_new(sjs, "url", js_url); + } else { + json_decref(js_url); + } + SCReturnPtr(sjs, "json_t"); } -// FLOWLOCK_UNLOCK(p->flow); - SCReturnInt(TM_ECODE_DONE); + json_decref(sjs); + SCReturnPtr(NULL, "json_t"); } -int JsonEmailLogger(ThreadVars *tv, void *thread_data, const Packet *p, Flow *f, void *state, void *tx, uint64_t tx_id) { - SCEnter(); - JsonEmailLogThread *jhl = (JsonEmailLogThread *)thread_data; - MemBuffer *buffer = (MemBuffer *)jhl->buffer; +/* JSON format logging */ +TmEcode JsonEmailLogJson(JsonEmailLogThread *aft, json_t *js, const Packet *p, Flow *f, void *state, void *vtx, uint64_t tx_id) +{ + json_t *sjs = JsonEmailLogJsonData(f, state, vtx, tx_id); + OutputJsonEmailCtx *email_ctx = aft->emaillog_ctx; + SMTPTransaction *tx = (SMTPTransaction *) vtx; + + if ((email_ctx->flags & LOG_EMAIL_EXTENDED) || (email_ctx->fields != 0)) + JsonEmailLogJSONCustom(email_ctx, sjs, tx); - json_t *js = CreateJSONHeader((Packet *)p, 1, "smtp"); - if (unlikely(js == NULL)) - return TM_ECODE_OK; +#ifdef HAVE_NSS + JsonEmailLogJSONMd5(email_ctx, sjs, tx); +#endif - /* reset */ - MemBufferReset(buffer); + if (sjs) { + json_object_set_new(js, "email", sjs); + SCReturnInt(TM_ECODE_OK); + } else + SCReturnInt(TM_ECODE_FAILED); +} + +json_t *JsonEmailAddMetadata(const Flow *f, uint32_t tx_id) +{ + SMTPState *smtp_state = (SMTPState *)FlowGetAppState(f); + if (smtp_state) { + SMTPTransaction *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_SMTP, smtp_state, tx_id); - if (JsonEmailLogJson(jhl, js, p, f, state, tx, tx_id) == TM_ECODE_OK) { - OutputJSONBuffer(js, jhl->emaillog_ctx->file_ctx, buffer); + if (tx) { + return JsonEmailLogJsonData(f, smtp_state, tx, tx_id); + } } - json_object_del(js, "smtp"); - json_object_clear(js); - json_decref(js); + return NULL; +} + + +void OutputEmailInitConf(ConfNode *conf, OutputJsonEmailCtx *email_ctx) +{ + if (conf) { + const char *extended = ConfNodeLookupChildValue(conf, "extended"); + + if (extended != NULL) { + if (ConfValIsTrue(extended)) { + email_ctx->flags = LOG_EMAIL_EXTENDED; + } + } + + email_ctx->fields = 0; + ConfNode *custom; + if ((custom = ConfNodeLookupChild(conf, "custom")) != NULL) { + ConfNode *field; + TAILQ_FOREACH(field, &custom->head, next) { + if (field != NULL) { + int f = 0; + while(email_fields[f].config_field) { + if ((strcmp(email_fields[f].config_field, + field->val) == 0) || + (strcasecmp(email_fields[f].email_field, + field->val) == 0)) + { + email_ctx->fields |= (1ULL<<f); + break; + } + f++; + } + } + } + } - SCReturnInt(TM_ECODE_OK); + email_ctx->flags = 0; + ConfNode *md5_conf; + if ((md5_conf = ConfNodeLookupChild(conf, "md5")) != NULL) { + ConfNode *field; + TAILQ_FOREACH(field, &md5_conf->head, next) { + if (field != NULL) { + if (strcmp("body", field->val) == 0) { + SCLogInfo("Going to log the md5 sum of email body"); + email_ctx->flags |= LOG_EMAIL_BODY_MD5; + } + if (strcmp("subject", field->val) == 0) { + SCLogInfo("Going to log the md5 sum of email subject"); + email_ctx->flags |= LOG_EMAIL_SUBJECT_MD5; + } + } + } + } + } + return; } + #endif diff --git a/framework/src/suricata/src/output-json-email-common.h b/framework/src/suricata/src/output-json-email-common.h index 7a95954c..88cfa557 100644 --- a/framework/src/suricata/src/output-json-email-common.h +++ b/framework/src/suricata/src/output-json-email-common.h @@ -27,14 +27,20 @@ typedef struct OutputJsonEmailCtx_ { LogFileCtx *file_ctx; uint32_t flags; /** Store mode */ + uint64_t fields;/** Store fields */ } OutputJsonEmailCtx; +#ifdef HAVE_LIBJANSSON typedef struct JsonEmailLogThread_ { OutputJsonEmailCtx *emaillog_ctx; MemBuffer *buffer; } JsonEmailLogThread; -int JsonEmailLogger(ThreadVars *tv, void *thread_data, const Packet *p, Flow *f, void *state, void *tx, uint64_t tx_id); +TmEcode JsonEmailLogJson(JsonEmailLogThread *aft, json_t *js, const Packet *p, Flow *f, void *state, void *vtx, uint64_t tx_id); +json_t *JsonEmailAddMetadata(const Flow *f, uint32_t tx_id); +#endif + +void OutputEmailInitConf(ConfNode *conf, OutputJsonEmailCtx *email_ctx); #endif /* __OUTPUT_JSON_EMAIL_COMMON_H__ */ diff --git a/framework/src/suricata/src/output-json-file.c b/framework/src/suricata/src/output-json-file.c index cbfa0c4d..9506a746 100644 --- a/framework/src/suricata/src/output-json-file.c +++ b/framework/src/suricata/src/output-json-file.c @@ -51,12 +51,15 @@ #include "util-buffer.h" #include "util-byte.h" -#include "output.h" -#include "output-json.h" - #include "log-file.h" #include "util-logopenfile.h" +#include "output.h" +#include "output-json.h" +#include "output-json-http.h" +#include "output-json-smtp.h" +#include "output-json-email-common.h" + #include "app-layer-htp.h" #include "util-memcmp.h" #include "stream-tcp-reassemble.h" @@ -74,99 +77,6 @@ typedef struct JsonFileLogThread_ { MemBuffer *buffer; } JsonFileLogThread; -static json_t *LogFileMetaGetUri(const Packet *p, const File *ff) -{ - HtpState *htp_state = (HtpState *)p->flow->alstate; - json_t *js = NULL; - if (htp_state != NULL) { - htp_tx_t *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP, htp_state, ff->txid); - if (tx != NULL) { - HtpTxUserData *tx_ud = htp_tx_get_user_data(tx); - if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) { - char *s = bstr_util_strdup_to_c(tx_ud->request_uri_normalized); - if (s != NULL) { - js = json_string(s); - SCFree(s); - if (js != NULL) - return js; - } - } - } - } - - return NULL; -} - -static json_t *LogFileMetaGetHost(const Packet *p, const File *ff) -{ - HtpState *htp_state = (HtpState *)p->flow->alstate; - json_t *js = NULL; - if (htp_state != NULL) { - htp_tx_t *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP, htp_state, ff->txid); - if (tx != NULL && tx->request_hostname != NULL) { - char *s = bstr_util_strdup_to_c(tx->request_hostname); - if (s != NULL) { - js = json_string(s); - SCFree(s); - if (js != NULL) - return js; - } - } - } - - return NULL; -} - -static json_t *LogFileMetaGetReferer(const Packet *p, const File *ff) -{ - HtpState *htp_state = (HtpState *)p->flow->alstate; - json_t *js = NULL; - if (htp_state != NULL) { - htp_tx_t *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP, htp_state, ff->txid); - if (tx != NULL) { - htp_header_t *h = NULL; - h = (htp_header_t *)htp_table_get_c(tx->request_headers, - "Referer"); - if (h != NULL) { - char *s = bstr_util_strdup_to_c(h->value); - if (s != NULL) { - js = json_string(s); - SCFree(s); - if (js != NULL) - return js; - } - } - } - } - - return NULL; -} - -static json_t *LogFileMetaGetUserAgent(const Packet *p, const File *ff) -{ - HtpState *htp_state = (HtpState *)p->flow->alstate; - json_t *js = NULL; - if (htp_state != NULL) { - htp_tx_t *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP, htp_state, ff->txid); - if (tx != NULL) { - htp_header_t *h = NULL; - h = (htp_header_t *)htp_table_get_c(tx->request_headers, - "User-Agent"); - if (h != NULL) { - char *s = bstr_util_strdup_to_c(h->value); - if (s != NULL) { - js = json_string(s); - SCFree(s); - if (js != NULL) - return js; - } - } - } - } - - return NULL; -} - /** * \internal * \brief Write meta data on a single line json record @@ -175,33 +85,32 @@ static void FileWriteJsonRecord(JsonFileLogThread *aft, const Packet *p, const F { MemBuffer *buffer = (MemBuffer *)aft->buffer; json_t *js = CreateJSONHeader((Packet *)p, 0, "fileinfo"); //TODO const + json_t *hjs = NULL; if (unlikely(js == NULL)) return; /* reset */ MemBufferReset(buffer); - json_t *hjs = json_object(); - if (unlikely(hjs == NULL)) { - json_decref(js); - return; - } - - json_object_set_new(hjs, "app_proto", json_string(AppProtoToString(p->flow->alproto))); switch (p->flow->alproto) { case ALPROTO_HTTP: - json_object_set_new(hjs, "url", LogFileMetaGetUri(p, ff)); - json_object_set_new(hjs, "hostname", LogFileMetaGetHost(p, ff)); - json_object_set_new(hjs, "http_refer", LogFileMetaGetReferer(p, ff)); - json_object_set_new(hjs, "http_user_agent", LogFileMetaGetUserAgent(p, ff)); - json_object_set_new(js, "http", hjs); + hjs = JsonHttpAddMetadata(p->flow, ff->txid); + if (hjs) + json_object_set_new(js, "http", hjs); + break; + case ALPROTO_SMTP: + hjs = JsonSMTPAddMetadata(p->flow, ff->txid); + if (hjs) + json_object_set_new(js, "smtp", hjs); + hjs = JsonEmailAddMetadata(p->flow, ff->txid); + if (hjs) + json_object_set_new(js, "email", hjs); break; } json_t *fjs = json_object(); if (unlikely(fjs == NULL)) { - json_decref(hjs); json_decref(js); return; } @@ -219,14 +128,11 @@ static void FileWriteJsonRecord(JsonFileLogThread *aft, const Packet *p, const F if (ff->flags & FILE_MD5) { size_t x; int i; - char *s = SCMalloc(256); - if (likely(s != NULL)) { - for (i = 0, x = 0; x < sizeof(ff->md5); x++) { - i += snprintf(&s[i], 255-i, "%02x", ff->md5[x]); - } - json_object_set_new(fjs, "md5", json_string(s)); - SCFree(s); + char s[256]; + for (i = 0, x = 0; x < sizeof(ff->md5); x++) { + i += snprintf(&s[i], 255-i, "%02x", ff->md5[x]); } + json_object_set_new(fjs, "md5", json_string(s)); } #endif break; @@ -252,7 +158,16 @@ static void FileWriteJsonRecord(JsonFileLogThread *aft, const Packet *p, const F json_object_set_new(js, "fileinfo", fjs); OutputJSONBuffer(js, aft->filelog_ctx->file_ctx, buffer); json_object_del(js, "fileinfo"); - json_object_del(js, "http"); + + switch (p->flow->alproto) { + case ALPROTO_HTTP: + json_object_del(js, "http"); + break; + case ALPROTO_SMTP: + json_object_del(js, "smtp"); + json_object_del(js, "email"); + break; + } json_object_clear(js); json_decref(js); diff --git a/framework/src/suricata/src/output-json-http.c b/framework/src/suricata/src/output-json-http.c index 31641985..68739873 100644 --- a/framework/src/suricata/src/output-json-http.c +++ b/framework/src/suricata/src/output-json-http.c @@ -362,9 +362,6 @@ static void JsonHttpLogJSON(JsonHttpLogThread *aft, json_t *js, htp_tx_t *tx, ui if (http_ctx->flags & LOG_HTTP_EXTENDED) JsonHttpLogJSONExtended(hjs, tx); - /* tx id for correlation with alerts */ - json_object_set_new(hjs, "tx_id", json_integer(tx_id)); - json_object_set_new(js, "http", hjs); } @@ -376,7 +373,7 @@ static int JsonHttpLogger(ThreadVars *tv, void *thread_data, const Packet *p, Fl JsonHttpLogThread *jhl = (JsonHttpLogThread *)thread_data; MemBuffer *buffer = (MemBuffer *)jhl->buffer; - json_t *js = CreateJSONHeader((Packet *)p, 1, "http"); //TODO const + json_t *js = CreateJSONHeaderWithTxId((Packet *)p, 1, "http", tx_id); //TODO const if (unlikely(js == NULL)) return TM_ECODE_OK; @@ -396,6 +393,27 @@ static int JsonHttpLogger(ThreadVars *tv, void *thread_data, const Packet *p, Fl SCReturnInt(TM_ECODE_OK); } +json_t *JsonHttpAddMetadata(const Flow *f, uint64_t tx_id) +{ + HtpState *htp_state = (HtpState *)FlowGetAppState(f); + if (htp_state) { + htp_tx_t *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP, htp_state, tx_id); + + if (tx) { + json_t *hjs = json_object(); + if (unlikely(hjs == NULL)) + return NULL; + + JsonHttpLogJSONBasic(hjs, tx); + JsonHttpLogJSONExtended(hjs, tx); + + return hjs; + } + } + + return NULL; +} + static void OutputHttpLogDeinit(OutputCtx *output_ctx) { LogHttpFileCtx *http_ctx = output_ctx->data; diff --git a/framework/src/suricata/src/output-json-http.h b/framework/src/suricata/src/output-json-http.h index ab412d22..0c886f3d 100644 --- a/framework/src/suricata/src/output-json-http.h +++ b/framework/src/suricata/src/output-json-http.h @@ -29,6 +29,7 @@ void TmModuleJsonHttpLogRegister (void); #ifdef HAVE_LIBJANSSON void JsonHttpLogJSONBasic(json_t *js, htp_tx_t *tx); void JsonHttpLogJSONExtended(json_t *js, htp_tx_t *tx); +json_t *JsonHttpAddMetadata(const Flow *f, uint64_t tx_id); #endif /* HAVE_LIBJANSSON */ #endif /* __OUTPUT_JSON_HTTP_H__ */ diff --git a/framework/src/suricata/src/output-json-smtp.c b/framework/src/suricata/src/output-json-smtp.c index f722c383..617b7247 100644 --- a/framework/src/suricata/src/output-json-smtp.c +++ b/framework/src/suricata/src/output-json-smtp.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2013 Open Information Security Foundation +/* Copyright (C) 2007-2015 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -54,11 +54,81 @@ #ifdef HAVE_LIBJANSSON #include <jansson.h> +static json_t *JsonSmtpDataLogger(const Flow *f, void *state, void *vtx, uint64_t tx_id) +{ + json_t *sjs = json_object(); + SMTPTransaction *tx = vtx; + SMTPString *rcptto_str; + if (sjs == NULL) { + return NULL; + } + if (((SMTPState *)state)->helo) { + json_object_set_new(sjs, "helo", + json_string((const char *)((SMTPState *)state)->helo)); + } + if (tx->mail_from) { + json_object_set_new(sjs, "mail_from", + json_string((const char *)tx->mail_from)); + } + if (!TAILQ_EMPTY(&tx->rcpt_to_list)) { + json_t *js_rcptto = json_array(); + if (likely(js_rcptto != NULL)) { + TAILQ_FOREACH(rcptto_str, &tx->rcpt_to_list, next) { + json_array_append_new(js_rcptto, json_string((char *)rcptto_str->str)); + } + json_object_set_new(sjs, "rcpt_to", js_rcptto); + } + } + + return sjs; +} + static int JsonSmtpLogger(ThreadVars *tv, void *thread_data, const Packet *p, Flow *f, void *state, void *tx, uint64_t tx_id) { SCEnter(); - int r = JsonEmailLogger(tv, thread_data, p, f, state, tx, tx_id); - SCReturnInt(r); + JsonEmailLogThread *jhl = (JsonEmailLogThread *)thread_data; + MemBuffer *buffer = (MemBuffer *)jhl->buffer; + + json_t *sjs; + json_t *js = CreateJSONHeaderWithTxId((Packet *)p, 1, "smtp", tx_id); + if (unlikely(js == NULL)) + return TM_ECODE_OK; + + /* reset */ + MemBufferReset(buffer); + + sjs = JsonSmtpDataLogger(f, state, tx, tx_id); + if (sjs) { + json_object_set_new(js, "smtp", sjs); + } + + if (JsonEmailLogJson(jhl, js, p, f, state, tx, tx_id) == TM_ECODE_OK) { + OutputJSONBuffer(js, jhl->emaillog_ctx->file_ctx, buffer); + } + json_object_del(js, "email"); + if (sjs) { + json_object_del(js, "smtp"); + } + + json_object_clear(js); + json_decref(js); + + SCReturnInt(TM_ECODE_OK); + +} + +json_t *JsonSMTPAddMetadata(const Flow *f, uint64_t tx_id) +{ + SMTPState *smtp_state = (SMTPState *)FlowGetAppState(f); + if (smtp_state) { + SMTPTransaction *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_SMTP, smtp_state, tx_id); + + if (tx) { + return JsonSmtpDataLogger(f, smtp_state, tx, tx_id); + } + } + + return NULL; } static void OutputSmtpLogDeInitCtx(OutputCtx *output_ctx) @@ -135,6 +205,8 @@ static OutputCtx *OutputSmtpLogInitSub(ConfNode *conf, OutputCtx *parent_ctx) email_ctx->file_ctx = ojc->file_ctx; + OutputEmailInitConf(conf, email_ctx); + output_ctx->data = email_ctx; output_ctx->DeInit = OutputSmtpLogDeInitCtxSub; @@ -154,7 +226,7 @@ static TmEcode JsonSmtpLogThreadInit(ThreadVars *t, void *initdata, void **data) if(initdata == NULL) { - SCLogDebug("Error getting context for HTTPLog. \"initdata\" argument NULL"); + SCLogDebug("Error getting context for SMTPLog. \"initdata\" argument NULL"); SCFree(aft); return TM_ECODE_FAILED; } diff --git a/framework/src/suricata/src/output-json-smtp.h b/framework/src/suricata/src/output-json-smtp.h index d38187c7..2f79d992 100644 --- a/framework/src/suricata/src/output-json-smtp.h +++ b/framework/src/suricata/src/output-json-smtp.h @@ -25,5 +25,8 @@ #define __OUTPUT_JSON_SMTP_H__ void TmModuleJsonSmtpLogRegister (void); +#ifdef HAVE_LIBJANSSON +json_t *JsonSMTPAddMetadata(const Flow *f, uint64_t tx_id); +#endif #endif /* __OUTPUT_JSON_SMTP_H__ */ diff --git a/framework/src/suricata/src/output-json-template.c b/framework/src/suricata/src/output-json-template.c index ca4a9378..d360e674 100644 --- a/framework/src/suricata/src/output-json-template.c +++ b/framework/src/suricata/src/output-json-template.c @@ -180,6 +180,10 @@ static TmEcode JsonTemplateLogThreadDeinit(ThreadVars *t, void *data) void TmModuleJsonTemplateLogRegister(void) { + if (ConfGetNode("app-layer.protocols.template") == NULL) { + return; + } + tmm_modules[TMM_JSONTEMPLATELOG].name = "JsonTemplateLog"; tmm_modules[TMM_JSONTEMPLATELOG].ThreadInit = JsonTemplateLogThreadInit; tmm_modules[TMM_JSONTEMPLATELOG].ThreadDeinit = JsonTemplateLogThreadDeinit; diff --git a/framework/src/suricata/src/output-json.c b/framework/src/suricata/src/output-json.c index 74289f1b..9cc9bd94 100644 --- a/framework/src/suricata/src/output-json.c +++ b/framework/src/suricata/src/output-json.c @@ -119,10 +119,6 @@ void OutputJsonRegisterTests (void) #define OUTPUT_BUFFER_SIZE 65535 -#ifndef OS_WIN32 -static int alert_syslog_level = DEFAULT_ALERT_SYSLOG_LEVEL; -#endif /* OS_WIN32 */ - TmEcode OutputJson (ThreadVars *, Packet *, void *, PacketQueue *, PacketQueue *); TmEcode OutputJsonThreadInit(ThreadVars *, void *, void **); TmEcode OutputJsonThreadDeinit(ThreadVars *, void *); @@ -327,48 +323,51 @@ json_t *CreateJSONHeader(Packet *p, int direction_sensitive, char *event_type) return js; } +json_t *CreateJSONHeaderWithTxId(Packet *p, int direction_sensitive, char *event_type, uint32_t tx_id) +{ + json_t *js = CreateJSONHeader(p, direction_sensitive, event_type); + if (unlikely(js == NULL)) + return NULL; + + /* tx id for correlation with other events */ + json_object_set_new(js, "tx_id", json_integer(tx_id)); + + return js; +} + +static int MemBufferCallback(const char *str, size_t size, void *data) +{ + MemBuffer *memb = data; +#if 0 // can't expand, need a MemBuffer ** + /* since we can have many threads, the buffer might not be big enough. + * * Expand if necessary. */ + if (MEMBUFFER_OFFSET(memb) + size > MEMBUFFER_SIZE(memb)) { + MemBufferExpand(&memb, OUTPUT_BUFFER_SIZE); + } +#endif + MemBufferWriteString(memb, "%s", str); + return 0; +} + int OutputJSONBuffer(json_t *js, LogFileCtx *file_ctx, MemBuffer *buffer) { - char *js_s = json_dumps(js, - JSON_PRESERVE_ORDER|JSON_COMPACT|JSON_ENSURE_ASCII| + if (file_ctx->sensor_name) { + json_object_set_new(js, "host", + json_string(file_ctx->sensor_name)); + } + + int r = json_dump_callback(js, MemBufferCallback, buffer, + JSON_PRESERVE_ORDER|JSON_COMPACT|JSON_ENSURE_ASCII| #ifdef JSON_ESCAPE_SLASH JSON_ESCAPE_SLASH #else 0 #endif ); - if (unlikely(js_s == NULL)) + if (r != 0) return TM_ECODE_OK; - SCMutexLock(&file_ctx->fp_mutex); - if (file_ctx->type == LOGFILE_TYPE_SYSLOG) - { - if (file_ctx->prefix != NULL) - { - syslog(alert_syslog_level, "%s%s", file_ctx->prefix, js_s); - } - else - { - syslog(alert_syslog_level, "%s", js_s); - } - } - else if (file_ctx->type == LOGFILE_TYPE_FILE || - file_ctx->type == LOGFILE_TYPE_UNIX_DGRAM || - file_ctx->type == LOGFILE_TYPE_UNIX_STREAM) - { - if (file_ctx->prefix != NULL) - { - MemBufferWriteString(buffer, "%s%s\n", file_ctx->prefix, js_s); - } - else - { - MemBufferWriteString(buffer, "%s\n", js_s); - } - file_ctx->Write((const char *)MEMBUFFER_BUFFER(buffer), - MEMBUFFER_OFFSET(buffer), file_ctx); - } - SCMutexUnlock(&file_ctx->fp_mutex); - free(js_s); + LogFileWrite(file_ctx, buffer); return 0; } @@ -425,6 +424,9 @@ void OutputJsonExitPrintStats(ThreadVars *tv, void *data) OutputCtx *OutputJsonInitCtx(ConfNode *conf) { OutputJsonCtx *json_ctx = SCCalloc(1, sizeof(OutputJsonCtx));; + + const char *sensor_name = ConfNodeLookupChildValue(conf, "sensor-name"); + if (unlikely(json_ctx == NULL)) { SCLogDebug("AlertJsonInitCtx: Could not create new LogFileCtx"); return NULL; @@ -437,6 +439,17 @@ OutputCtx *OutputJsonInitCtx(ConfNode *conf) return NULL; } + if (sensor_name) { + json_ctx->file_ctx->sensor_name = SCStrdup(sensor_name); + if (json_ctx->file_ctx->sensor_name == NULL) { + LogFileFreeCtx(json_ctx->file_ctx); + SCFree(json_ctx); + return NULL; + } + } else { + json_ctx->file_ctx->sensor_name = NULL; + } + OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx)); if (unlikely(output_ctx == NULL)) { LogFileFreeCtx(json_ctx->file_ctx); @@ -465,6 +478,14 @@ OutputCtx *OutputJsonInitCtx(ConfNode *conf) json_ctx->json_out = LOGFILE_TYPE_UNIX_DGRAM; } else if (strcmp(output_s, "unix_stream") == 0) { json_ctx->json_out = LOGFILE_TYPE_UNIX_STREAM; + } else if (strcmp(output_s, "redis") == 0) { +#ifdef HAVE_LIBHIREDIS + json_ctx->json_out = LOGFILE_TYPE_REDIS; +#else + SCLogError(SC_ERR_INVALID_ARGUMENT, + "redis JSON output option is not compiled"); + exit(EXIT_FAILURE); +#endif } else { SCLogError(SC_ERR_INVALID_ARGUMENT, "Invalid JSON output option: %s", output_s); @@ -526,7 +547,7 @@ OutputCtx *OutputJsonInitCtx(ConfNode *conf) if (level_s != NULL) { int level = SCMapEnumNameToValue(level_s, SCSyslogGetLogLevelMap()); if (level != -1) { - alert_syslog_level = level; + json_ctx->file_ctx->syslog_setup.alert_syslog_level = level; } } @@ -537,6 +558,29 @@ OutputCtx *OutputJsonInitCtx(ConfNode *conf) openlog(ident, LOG_PID|LOG_NDELAY, facility); } +#ifdef HAVE_LIBHIREDIS + else if (json_ctx->json_out == LOGFILE_TYPE_REDIS) { + ConfNode *redis_node = ConfNodeLookupChild(conf, "redis"); + if (!json_ctx->file_ctx->sensor_name) { + char hostname[1024]; + gethostname(hostname, 1023); + json_ctx->file_ctx->sensor_name = SCStrdup(hostname); + } + if (json_ctx->file_ctx->sensor_name == NULL) { + LogFileFreeCtx(json_ctx->file_ctx); + SCFree(json_ctx); + SCFree(output_ctx); + return NULL; + } + + if (SCConfLogOpenRedis(redis_node, json_ctx->file_ctx) < 0) { + LogFileFreeCtx(json_ctx->file_ctx); + SCFree(json_ctx); + SCFree(output_ctx); + return NULL; + } + } +#endif const char *sensor_id_s = ConfNodeLookupChildValue(conf, "sensor-id"); if (sensor_id_s != NULL) { diff --git a/framework/src/suricata/src/output-json.h b/framework/src/suricata/src/output-json.h index 1acde3e6..89e11d86 100644 --- a/framework/src/suricata/src/output-json.h +++ b/framework/src/suricata/src/output-json.h @@ -35,11 +35,11 @@ void TmModuleOutputJsonRegister (void); void CreateJSONFlowId(json_t *js, const Flow *f); void JsonTcpFlags(uint8_t flags, json_t *js); json_t *CreateJSONHeader(Packet *p, int direction_sensative, char *event_type); +json_t *CreateJSONHeaderWithTxId(Packet *p, int direction_sensitive, char *event_type, uint32_t tx_id); TmEcode OutputJSON(json_t *js, void *data, uint64_t *count); int OutputJSONBuffer(json_t *js, LogFileCtx *file_ctx, MemBuffer *buffer); OutputCtx *OutputJsonInitCtx(ConfNode *); - enum JsonFormat { COMPACT, INDENT }; /* diff --git a/framework/src/suricata/src/output-lua.c b/framework/src/suricata/src/output-lua.c index fc5a5621..a8fcb246 100644 --- a/framework/src/suricata/src/output-lua.c +++ b/framework/src/suricata/src/output-lua.c @@ -40,6 +40,8 @@ #include "output.h" #include "app-layer-htp.h" #include "app-layer.h" +#include "app-layer-ssl.h" +#include "app-layer-ssh.h" #include "app-layer-parser.h" #include "util-privs.h" #include "util-buffer.h" @@ -57,6 +59,8 @@ #include "util-lua-common.h" #include "util-lua-http.h" #include "util-lua-dns.h" +#include "util-lua-tls.h" +#include "util-lua-ssh.h" #define MODULE_NAME "LuaLog" @@ -229,6 +233,166 @@ static int LuaPacketConditionAlerts(ThreadVars *tv, const Packet *p) } /** \internal + * \brief Packet Logger for lua scripts, for tls + * + * A single call to this function will run one script for a single + * packet. If it is called, it means that the registered condition + * function has returned TRUE. + * + * The script is called once for each packet. + */ +static int LuaPacketLoggerTls(ThreadVars *tv, void *thread_data, const Packet *p) +{ + LogLuaThreadCtx *td = (LogLuaThreadCtx *)thread_data; + + char timebuf[64]; + CreateTimeString(&p->ts, timebuf, sizeof(timebuf)); + + SCMutexLock(&td->lua_ctx->m); + + lua_getglobal(td->lua_ctx->luastate, "log"); + + LuaStateSetThreadVars(td->lua_ctx->luastate, tv); + LuaStateSetPacket(td->lua_ctx->luastate, (Packet *)p); + LuaStateSetFlow(td->lua_ctx->luastate, p->flow, /* unlocked */LUA_FLOW_NOT_LOCKED_BY_PARENT); + + int retval = lua_pcall(td->lua_ctx->luastate, 0, 0, 0); + if (retval != 0) { + SCLogInfo("failed to run script: %s", lua_tostring(td->lua_ctx->luastate, -1)); + } + + SCMutexUnlock(&td->lua_ctx->m); + FLOWLOCK_WRLOCK(p->flow); + + SSLState *ssl_state = (SSLState *)FlowGetAppState(p->flow); + if (ssl_state != NULL) + ssl_state->flags |= SSL_AL_FLAG_STATE_LOGGED_LUA; + + FLOWLOCK_UNLOCK(p->flow); + SCReturnInt(0); +} + +static int LuaPacketConditionTls(ThreadVars *tv, const Packet *p) +{ + if (p->flow == NULL) { + return FALSE; + } + + if (!(PKT_IS_IPV4(p)) && !(PKT_IS_IPV6(p))) { + return FALSE; + } + + if (!(PKT_IS_TCP(p))) { + return FALSE; + } + + FLOWLOCK_RDLOCK(p->flow); + uint16_t proto = FlowGetAppProtocol(p->flow); + if (proto != ALPROTO_TLS) + goto dontlog; + + SSLState *ssl_state = (SSLState *)FlowGetAppState(p->flow); + if (ssl_state == NULL) { + SCLogDebug("no tls state, so no request logging"); + goto dontlog; + } + + if (ssl_state->server_connp.cert0_issuerdn == NULL || + ssl_state->server_connp.cert0_subject == NULL) + goto dontlog; + + /* We only log the state once */ + if (ssl_state->flags & SSL_AL_FLAG_STATE_LOGGED_LUA) + goto dontlog; + + FLOWLOCK_UNLOCK(p->flow); + return TRUE; +dontlog: + FLOWLOCK_UNLOCK(p->flow); + return FALSE; +} + +/** \internal + * \brief Packet Logger for lua scripts, for ssh + * + * A single call to this function will run one script for a single + * packet. If it is called, it means that the registered condition + * function has returned TRUE. + * + * The script is called once for each packet. + */ +static int LuaPacketLoggerSsh(ThreadVars *tv, void *thread_data, const Packet *p) +{ + LogLuaThreadCtx *td = (LogLuaThreadCtx *)thread_data; + + char timebuf[64]; + CreateTimeString(&p->ts, timebuf, sizeof(timebuf)); + + SCMutexLock(&td->lua_ctx->m); + + lua_getglobal(td->lua_ctx->luastate, "log"); + + LuaStateSetThreadVars(td->lua_ctx->luastate, tv); + LuaStateSetPacket(td->lua_ctx->luastate, (Packet *)p); + LuaStateSetFlow(td->lua_ctx->luastate, p->flow, /* unlocked */LUA_FLOW_NOT_LOCKED_BY_PARENT); + + int retval = lua_pcall(td->lua_ctx->luastate, 0, 0, 0); + if (retval != 0) { + SCLogInfo("failed to run script: %s", lua_tostring(td->lua_ctx->luastate, -1)); + } + + SCMutexUnlock(&td->lua_ctx->m); + FLOWLOCK_WRLOCK(p->flow); + + SshState *ssh_state = (SshState *)FlowGetAppState(p->flow); + if (ssh_state != NULL) + ssh_state->cli_hdr.flags |= SSH_FLAG_STATE_LOGGED_LUA; + + FLOWLOCK_UNLOCK(p->flow); + SCReturnInt(0); +} + +static int LuaPacketConditionSsh(ThreadVars *tv, const Packet *p) +{ + if (p->flow == NULL) { + return FALSE; + } + + if (!(PKT_IS_IPV4(p)) && !(PKT_IS_IPV6(p))) { + return FALSE; + } + + if (!(PKT_IS_TCP(p))) { + return FALSE; + } + + FLOWLOCK_RDLOCK(p->flow); + uint16_t proto = FlowGetAppProtocol(p->flow); + if (proto != ALPROTO_SSH) + goto dontlog; + + SshState *ssh_state = (SshState *)FlowGetAppState(p->flow); + if (ssh_state == NULL) { + SCLogDebug("no ssh state, so no request logging"); + goto dontlog; + } + + if (ssh_state->cli_hdr.software_version == NULL || + ssh_state->srv_hdr.software_version == NULL) + goto dontlog; + + /* We only log the state once */ + if (ssh_state->cli_hdr.flags & SSH_FLAG_STATE_LOGGED_LUA) + goto dontlog; + + FLOWLOCK_UNLOCK(p->flow); + return TRUE; +dontlog: + FLOWLOCK_UNLOCK(p->flow); + return FALSE; +} + +/** \internal * \brief Packet Logger for lua scripts, for packets * * A single call to this function will run one script for a single @@ -517,6 +681,10 @@ static int LuaScriptInit(const char *filename, LogLuaScriptOptions *options) { options->alproto = ALPROTO_HTTP; else if (strcmp(k,"protocol") == 0 && strcmp(v, "dns") == 0) options->alproto = ALPROTO_DNS; + else if (strcmp(k,"protocol") == 0 && strcmp(v, "tls") == 0) + options->alproto = ALPROTO_TLS; + else if (strcmp(k,"protocol") == 0 && strcmp(v, "ssh") == 0) + options->alproto = ALPROTO_SSH; else if (strcmp(k, "type") == 0 && strcmp(v, "packet") == 0) options->packet = 1; else if (strcmp(k, "filter") == 0 && strcmp(v, "alerts") == 0) @@ -617,6 +785,8 @@ static lua_State *LuaScriptSetup(const char *filename) * if the tx is registered in the state at runtime though. */ LuaRegisterHttpFunctions(luastate); LuaRegisterDnsFunctions(luastate); + LuaRegisterTlsFunctions(luastate); + LuaRegisterSshFunctions(luastate); if (lua_pcall(luastate, 0, 0, 0) != 0) { SCLogError(SC_ERR_LUA_ERROR, "couldn't run script 'setup' function: %s", lua_tostring(luastate, -1)); @@ -760,11 +930,17 @@ static OutputCtx *OutputLuaLogInit(ConfNode *conf) om->TxLogFunc = LuaTxLogger; om->alproto = ALPROTO_HTTP; AppLayerParserRegisterLogger(IPPROTO_TCP, ALPROTO_HTTP); + } else if (opts.alproto == ALPROTO_TLS) { + om->PacketLogFunc = LuaPacketLoggerTls; + om->PacketConditionFunc = LuaPacketConditionTls; } else if (opts.alproto == ALPROTO_DNS) { om->TxLogFunc = LuaTxLogger; om->alproto = ALPROTO_DNS; AppLayerParserRegisterLogger(IPPROTO_TCP, ALPROTO_DNS); AppLayerParserRegisterLogger(IPPROTO_UDP, ALPROTO_DNS); + } else if (opts.alproto == ALPROTO_SSH) { + om->PacketLogFunc = LuaPacketLoggerSsh; + om->PacketConditionFunc = LuaPacketConditionSsh; } else if (opts.packet && opts.alerts) { om->PacketLogFunc = LuaPacketLoggerAlerts; om->PacketConditionFunc = LuaPacketConditionAlerts; diff --git a/framework/src/suricata/src/runmode-pfring.c b/framework/src/suricata/src/runmode-pfring.c index fbbd8c91..f08bdad8 100644 --- a/framework/src/suricata/src/runmode-pfring.c +++ b/framework/src/suricata/src/runmode-pfring.c @@ -109,6 +109,7 @@ void *OldParsePfringConfig(const char *iface) } strlcpy(pfconf->iface, iface, sizeof(pfconf->iface)); + pfconf->flags = 0; pfconf->threads = 1; pfconf->cluster_id = 1; #ifdef HAVE_PFRING @@ -143,6 +144,7 @@ void *OldParsePfringConfig(const char *iface) SCLogError(SC_ERR_INVALID_ARGUMENT,"Could not get cluster-id from config"); } else { pfconf->cluster_id = (uint16_t)atoi(tmpclusterid); + pfconf->flags |= PFRING_CONF_FLAGS_CLUSTER; SCLogDebug("Going to use cluster-id %" PRId32, pfconf->cluster_id); } @@ -263,6 +265,7 @@ void *ParsePfringConfig(const char *iface) /* command line value has precedence */ if (ConfGet("pfring.cluster-id", &tmpclusterid) == 1) { pfconf->cluster_id = (uint16_t)atoi(tmpclusterid); + pfconf->flags |= PFRING_CONF_FLAGS_CLUSTER; SCLogDebug("Going to use command-line provided cluster-id %" PRId32, pfconf->cluster_id); } else { @@ -278,6 +281,7 @@ void *ParsePfringConfig(const char *iface) "Could not get cluster-id from config"); } else { pfconf->cluster_id = (uint16_t)atoi(tmpclusterid); + pfconf->flags |= PFRING_CONF_FLAGS_CLUSTER; SCLogDebug("Going to use cluster-id %" PRId32, pfconf->cluster_id); } } diff --git a/framework/src/suricata/src/runmode-unittests.c b/framework/src/suricata/src/runmode-unittests.c index 4f3fc8bf..3701d29a 100644 --- a/framework/src/suricata/src/runmode-unittests.c +++ b/framework/src/suricata/src/runmode-unittests.c @@ -277,6 +277,7 @@ void RunUnittests(int list_unittests, char *regex_arg) CudaBufferRegisterUnittests(); #endif AppLayerUnittestsRegister(); + MimeDecRegisterTests(); if (list_unittests) { UtListTests(regex_arg); } else { diff --git a/framework/src/suricata/src/source-pfring.c b/framework/src/suricata/src/source-pfring.c index 3b2b52df..527086f5 100644 --- a/framework/src/suricata/src/source-pfring.c +++ b/framework/src/suricata/src/source-pfring.c @@ -125,6 +125,10 @@ static SCMutex pfring_bpf_set_filter_lock = SCMUTEX_INITIALIZER; #define LIBPFRING_REENTRANT 0 #define LIBPFRING_WAIT_FOR_INCOMING 1 +typedef enum { + PFRING_FLAGS_ZERO_COPY = 0x1 +} PfringThreadVarsFlags; + /** * \brief Structure to hold thread specific variables. */ @@ -140,6 +144,8 @@ typedef struct PfringThreadVars_ uint16_t capture_kernel_packets; uint16_t capture_kernel_drops; + uint32_t flags; + ThreadVars *tv; TmSlot *slot; @@ -295,7 +301,8 @@ TmEcode ReceivePfringLoop(ThreadVars *tv, void *data, void *slot) struct pfring_pkthdr hdr; TmSlot *s = (TmSlot *)slot; time_t last_dump = 0; - struct timeval current_time; + u_int buffer_size; + u_char *pkt_buffer; ptv->slot = s->slot_next; @@ -325,16 +332,23 @@ TmEcode ReceivePfringLoop(ThreadVars *tv, void *data, void *slot) /* Some flavours of PF_RING may fail to set timestamp - see PF-RING-enabled libpcap code*/ hdr.ts.tv_sec = hdr.ts.tv_usec = 0; + /* Check for Zero-copy mode */ + if (ptv->flags & PFRING_FLAGS_ZERO_COPY) { + buffer_size = 0; + pkt_buffer = NULL; + } else { + buffer_size = GET_PKT_DIRECT_MAX_SIZE(p); + pkt_buffer = GET_PKT_DIRECT_DATA(p); + } + /* Depending on what compile time options are used for pfring we either return 0 or -1 on error and always 1 for success */ - u_char *pkt_buffer = GET_PKT_DIRECT_DATA(p); - u_int buffer_size = GET_PKT_DIRECT_MAX_SIZE(p); int r = pfring_recv(ptv->pd, &pkt_buffer, buffer_size, &hdr, LIBPFRING_WAIT_FOR_INCOMING); - /* Check for Zero-copy if buffer size is zero */ - if (buffer_size == 0) { + /* Check for Zero-copy mode */ + if (ptv->flags & PFRING_FLAGS_ZERO_COPY) { PacketSetData(p, pkt_buffer, hdr.caplen); } @@ -350,10 +364,9 @@ TmEcode ReceivePfringLoop(ThreadVars *tv, void *data, void *slot) } /* Trigger one dump of stats every second */ - TimeGet(¤t_time); - if (current_time.tv_sec != last_dump) { + if (p->ts.tv_sec != last_dump) { PfringDumpCounters(ptv); - last_dump = current_time.tv_sec; + last_dump = p->ts.tv_sec; } } else { SCLogError(SC_ERR_PF_RING_RECV,"pfring_recv error %" PRId32 "", r); @@ -386,7 +399,7 @@ TmEcode ReceivePfringThreadInit(ThreadVars *tv, void *initdata, void **data) u_int32_t version = 0; PfringIfaceConfig *pfconf = (PfringIfaceConfig *) initdata; unsigned int opflag; - + char const *active_runmode = RunmodeGetActive(); if (pfconf == NULL) return TM_ECODE_FAILED; @@ -415,9 +428,15 @@ TmEcode ReceivePfringThreadInit(ThreadVars *tv, void *initdata, void **data) SCReturnInt(TM_ECODE_FAILED); } + /* enable zero-copy mode for workers runmode */ + if (active_runmode && strcmp("workers", active_runmode) == 0) { + ptv->flags |= PFRING_FLAGS_ZERO_COPY; + SCLogInfo("Enabling zero-copy for %s", ptv->interface); + } + ptv->checksum_mode = pfconf->checksum_mode; - opflag = PF_RING_REENTRANT | PF_RING_PROMISC; + opflag = PF_RING_PROMISC; /* if suri uses VLAN and if we have a recent kernel, we need * to use parsed_pkt to get VLAN info */ @@ -466,8 +485,11 @@ TmEcode ReceivePfringThreadInit(ThreadVars *tv, void *initdata, void **data) if (rc != 0) { SCLogError(SC_ERR_PF_RING_SET_CLUSTER_FAILED, "pfring_set_cluster " "returned %d for cluster-id: %d", rc, ptv->cluster_id); - pfconf->DerefFunc(pfconf); - return TM_ECODE_FAILED; + if (rc != PF_RING_ERROR_NOT_SUPPORTED || (pfconf->flags & PFRING_CONF_FLAGS_CLUSTER)) { + /* cluster is mandatory as explicitly specified in the configuration */ + pfconf->DerefFunc(pfconf); + return TM_ECODE_FAILED; + } } } diff --git a/framework/src/suricata/src/source-pfring.h b/framework/src/suricata/src/source-pfring.h index e0878454..9871f458 100644 --- a/framework/src/suricata/src/source-pfring.h +++ b/framework/src/suricata/src/source-pfring.h @@ -31,8 +31,14 @@ #include <pfring.h> #endif +typedef enum { + PFRING_CONF_FLAGS_CLUSTER = 0x1 +} PfringIfaceConfigFlags; + typedef struct PfringIfaceConfig_ { + uint32_t flags; + /* cluster param */ int cluster_id; #ifdef HAVE_PFRING diff --git a/framework/src/suricata/src/stream-tcp-reassemble.c b/framework/src/suricata/src/stream-tcp-reassemble.c index caf955af..a947fab3 100644 --- a/framework/src/suricata/src/stream-tcp-reassemble.c +++ b/framework/src/suricata/src/stream-tcp-reassemble.c @@ -2620,51 +2620,58 @@ static inline int DoReassemble(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, TcpSegment *seg, ReassembleData *rd, Packet *p) { - /* fast path 1: segment is exactly what we need */ - if (likely(rd->data_len == 0 && - SEQ_EQ(seg->seq, rd->ra_base_seq+1) && - SEQ_EQ(stream->last_ack, (seg->seq + seg->payload_len)))) - { - /* process single segment directly */ - AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream, - seg->payload, seg->payload_len, - StreamGetAppLayerFlags(ssn, stream, p)); - AppLayerProfilingStore(ra_ctx->app_tctx, p); - rd->data_sent += seg->payload_len; - rd->ra_base_seq += seg->payload_len; + /* fast paths: send data directly into the app layer, w/o first doing + * a copy step. However, don't use the fast path until protocol detection + * has been completed + * TODO if initial data is big enough for proto detect, we could do the + * fast path anyway. */ + if (stream->flags & STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_COMPLETED) { + /* fast path 1: segment is exactly what we need */ + if (likely(rd->data_len == 0 && + SEQ_EQ(seg->seq, rd->ra_base_seq+1) && + SEQ_EQ(stream->last_ack, (seg->seq + seg->payload_len)))) + { + /* process single segment directly */ + AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream, + seg->payload, seg->payload_len, + StreamGetAppLayerFlags(ssn, stream, p)); + AppLayerProfilingStore(ra_ctx->app_tctx, p); + rd->data_sent += seg->payload_len; + rd->ra_base_seq += seg->payload_len; #ifdef DEBUG - ra_ctx->fp1++; + ra_ctx->fp1++; #endif - /* if after the first data chunk we have no alproto yet, - * there is no point in continueing here. */ - if (!StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(stream)) { - SCLogDebug("no alproto after first data chunk"); - return 0; - } - return 1; - /* fast path 2: segment acked completely, meets minimal size req for 0copy processing */ - } else if (rd->data_len == 0 && - SEQ_EQ(seg->seq, rd->ra_base_seq+1) && - SEQ_GT(stream->last_ack, (seg->seq + seg->payload_len)) && - seg->payload_len >= stream_config.zero_copy_size) - { - /* process single segment directly */ - AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream, - seg->payload, seg->payload_len, - StreamGetAppLayerFlags(ssn, stream, p)); - AppLayerProfilingStore(ra_ctx->app_tctx, p); - rd->data_sent += seg->payload_len; - rd->ra_base_seq += seg->payload_len; + /* if after the first data chunk we have no alproto yet, + * there is no point in continueing here. */ + if (!StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(stream)) { + SCLogDebug("no alproto after first data chunk"); + return 0; + } + return 1; + /* fast path 2: segment acked completely, meets minimal size req for 0copy processing */ + } else if (rd->data_len == 0 && + SEQ_EQ(seg->seq, rd->ra_base_seq+1) && + SEQ_GT(stream->last_ack, (seg->seq + seg->payload_len)) && + seg->payload_len >= stream_config.zero_copy_size) + { + /* process single segment directly */ + AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream, + seg->payload, seg->payload_len, + StreamGetAppLayerFlags(ssn, stream, p)); + AppLayerProfilingStore(ra_ctx->app_tctx, p); + rd->data_sent += seg->payload_len; + rd->ra_base_seq += seg->payload_len; #ifdef DEBUG - ra_ctx->fp2++; + ra_ctx->fp2++; #endif - /* if after the first data chunk we have no alproto yet, - * there is no point in continueing here. */ - if (!StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(stream)) { - SCLogDebug("no alproto after first data chunk"); - return 0; + /* if after the first data chunk we have no alproto yet, + * there is no point in continueing here. */ + if (!StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(stream)) { + SCLogDebug("no alproto after first data chunk"); + return 0; + } + return 1; } - return 1; } #ifdef DEBUG ra_ctx->sp++; @@ -2888,6 +2895,49 @@ int StreamTcpReassembleAppLayer (ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, GetSessionSize(ssn, p); #endif + /* Check if we have a gap at the start of the stream. 2 conditions: + * 1. no segments, but last_ack moved fwd + * 2. segments, but clearly some missing: if last_ack is + * bigger than the list start and the list start is bigger than + * next_seq, we know we are missing data that has been ack'd. That + * won't get retransmitted, so it's a data gap. + */ + if (!(ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED)) { + int ackadd = (ssn->state >= TCP_FIN_WAIT2) ? 2 : 1; + if ((stream->seg_list == NULL && /*1*/ + stream->ra_app_base_seq == stream->isn && + SEQ_GT(stream->last_ack, stream->isn + ackadd)) + || + (stream->seg_list != NULL && /*2*/ + SEQ_GT(stream->seg_list->seq, stream->ra_app_base_seq+1) && + SEQ_LT(stream->seg_list->seq, stream->last_ack))) + { + if (stream->seg_list == NULL) { + SCLogDebug("no segs, last_ack moved fwd so GAP " + "(base %u, isn %u, last_ack %u => diff %u) p %"PRIu64, + stream->ra_app_base_seq, stream->isn, stream->last_ack, + stream->last_ack - (stream->isn + ackadd), p->pcap_cnt); + } + + /* send gap signal */ + AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream, + NULL, 0, + StreamGetAppLayerFlags(ssn, stream, p)|STREAM_GAP); + AppLayerProfilingStore(ra_ctx->app_tctx, p); + + /* set a GAP flag and make sure not bothering this stream anymore */ + SCLogDebug("STREAMTCP_STREAM_FLAG_GAP set"); + stream->flags |= STREAMTCP_STREAM_FLAG_GAP; + + StreamTcpSetEvent(p, STREAM_REASSEMBLY_SEQ_GAP); + StatsIncr(tv, ra_ctx->counter_tcp_reass_gap); +#ifdef DEBUG + dbg_app_layer_gap++; +#endif + SCReturnInt(0); + } + } + /* if no segments are in the list or all are already processed, * and state is beyond established, we send an empty msg */ TcpSegment *seg_tail = stream->seg_list_tail; @@ -2934,35 +2984,14 @@ int StreamTcpReassembleAppLayer (ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, /* loop through the segments and fill one or more msgs */ TcpSegment *seg = stream->seg_list; SCLogDebug("pre-loop seg %p", seg); - - /* Check if we have a gap at the start of the list. If last_ack is - * bigger than the list start and the list start is bigger than - * next_seq, we know we are missing data that has been ack'd. That - * won't get retransmitted, so it's a data gap. - */ - if (!(ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED)) { - if (SEQ_GT(seg->seq, next_seq) && SEQ_LT(seg->seq, stream->last_ack)) { - /* send gap signal */ - AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream, - NULL, 0, - StreamGetAppLayerFlags(ssn, stream, p)|STREAM_GAP); - AppLayerProfilingStore(ra_ctx->app_tctx, p); - - /* set a GAP flag and make sure not bothering this stream anymore */ - SCLogDebug("STREAMTCP_STREAM_FLAG_GAP set"); - stream->flags |= STREAMTCP_STREAM_FLAG_GAP; - - StreamTcpSetEvent(p, STREAM_REASSEMBLY_SEQ_GAP); - StatsIncr(tv, ra_ctx->counter_tcp_reass_gap); -#ifdef DEBUG - dbg_app_layer_gap++; +#ifdef DEBUG_VALIDATION + uint64_t bytes = 0; #endif - SCReturnInt(0); - } - } - for (; seg != NULL; ) { +#ifdef DEBUG_VALIDATION + bytes += seg->payload_len; +#endif /* if in inline mode, we process all segments regardless of whether * they are ack'd or not. In non-inline, we process only those that * are at least partly ack'd. */ @@ -3007,6 +3036,9 @@ int StreamTcpReassembleAppLayer (ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, } seg = next_seg; } +#ifdef DEBUG_VALIDATION /* we should never have this much data queued */ + BUG_ON(bytes > 1000000ULL && bytes > (stream->window * 1.5)); +#endif /* put the partly filled smsg in the queue to the l7 handler */ if (rd.data_len > 0) { @@ -3035,6 +3067,8 @@ int StreamTcpReassembleAppLayer (ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, } else { TcpSegment *tmp_seg = stream->seg_list; while (tmp_seg != NULL) { + if (!(tmp_seg->flags & SEGMENTTCP_FLAG_APPLAYER_PROCESSED)) + break; tmp_seg->flags &= ~SEGMENTTCP_FLAG_APPLAYER_PROCESSED; tmp_seg = tmp_seg->next; } @@ -5417,6 +5451,9 @@ static int StreamTcpReassembleTest30 (void) th_flag = TH_ACK|TH_PUSH; th_flags = TH_ACK; + ssn.client.last_ack = 2; + ssn.client.isn = 1; + ssn.server.last_ack = 22; ssn.server.ra_raw_base_seq = ssn.server.ra_app_base_seq = 9; ssn.server.isn = 9; @@ -6100,7 +6137,7 @@ static int StreamTcpReassembleTest38 (void) ssn.server.last_ack = 60; ssn.client.ra_raw_base_seq = ssn.client.ra_app_base_seq = 9; ssn.client.isn = 9; - ssn.client.last_ack = 60; + ssn.client.last_ack = 9; f.alproto = ALPROTO_UNKNOWN; f.flags |= FLOW_IPV4; @@ -7218,7 +7255,7 @@ static int StreamTcpReassembleTest45 (void) ssn.server.last_ack = 60; STREAMTCP_SET_RA_BASE_SEQ(&ssn.client, 9); ssn.client.isn = 9; - ssn.client.last_ack = 60; + ssn.client.last_ack = 9; f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 200, 220); if (f == NULL) @@ -7337,7 +7374,7 @@ static int StreamTcpReassembleTest46 (void) ssn.server.next_seq = ssn.server.isn; STREAMTCP_SET_RA_BASE_SEQ(&ssn.client, 9); ssn.client.isn = 9; - ssn.client.last_ack = 60; + ssn.client.last_ack = 9; ssn.client.next_seq = ssn.client.isn; f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 200, 220); @@ -8405,6 +8442,9 @@ static int StreamTcpReassembleInlineTest10(void) StreamTcpUTInitInline(); StreamTcpUTSetupSession(&ssn); StreamTcpUTSetupStream(&ssn.server, 1); + ssn.server.last_ack = 2; + StreamTcpUTSetupStream(&ssn.client, 1); + ssn.client.last_ack = 2; f = UTHBuildFlow(AF_INET, "1.1.1.1", "2.2.2.2", 1024, 80); if (f == NULL) diff --git a/framework/src/suricata/src/stream-tcp.c b/framework/src/suricata/src/stream-tcp.c index 6cde8651..9dce7070 100644 --- a/framework/src/suricata/src/stream-tcp.c +++ b/framework/src/suricata/src/stream-tcp.c @@ -764,8 +764,13 @@ uint32_t StreamTcpGetStreamSize(TcpStream *stream) #define StreamTcpUpdateLastAck(ssn, stream, ack) { \ if (SEQ_GT((ack), (stream)->last_ack)) \ { \ + SCLogDebug("ssn %p: last_ack set to %"PRIu32", moved %u forward", (ssn), (ack), (ack) - (stream)->last_ack); \ + if ((SEQ_LEQ((stream)->last_ack, (stream)->next_seq) && SEQ_GT((ack),(stream)->next_seq))) { \ + SCLogDebug("last_ack just passed next_seq: %u (was %u) > %u", (ack), (stream)->last_ack, (stream)->next_seq); \ + } else { \ + SCLogDebug("next_seq (%u) <> last_ack now %d", (stream)->next_seq, (int)(stream)->next_seq - (ack)); \ + }\ (stream)->last_ack = (ack); \ - SCLogDebug("ssn %p: last_ack set to %"PRIu32, (ssn), (stream)->last_ack); \ StreamTcpSackPruneList((stream)); \ } else { \ SCLogDebug("ssn %p: no update: ack %u, last_ack %"PRIu32", next_seq %u (state %u)", \ @@ -793,6 +798,13 @@ static int StreamTcpPacketIsRetransmission(TcpStream *stream, Packet *p) if (p->payload_len == 0) SCReturnInt(0); + /* retransmission of already partially ack'd data */ + if (SEQ_LT(TCP_GET_SEQ(p), stream->last_ack) && SEQ_GT((TCP_GET_SEQ(p) + p->payload_len), stream->last_ack)) + { + StreamTcpSetEvent(p, STREAM_PKT_RETRANSMISSION); + SCReturnInt(1); + } + /* retransmission of already ack'd data */ if (SEQ_LEQ((TCP_GET_SEQ(p) + p->payload_len), stream->last_ack)) { StreamTcpSetEvent(p, STREAM_PKT_RETRANSMISSION); @@ -805,8 +817,8 @@ static int StreamTcpPacketIsRetransmission(TcpStream *stream, Packet *p) SCReturnInt(2); } - SCLogDebug("seq %u payload_len %u => %u, last_ack %u", TCP_GET_SEQ(p), - p->payload_len, (TCP_GET_SEQ(p) + p->payload_len), stream->last_ack); + SCLogDebug("seq %u payload_len %u => %u, last_ack %u, next_seq %u", TCP_GET_SEQ(p), + p->payload_len, (TCP_GET_SEQ(p) + p->payload_len), stream->last_ack, stream->next_seq); SCReturnInt(0); } @@ -2040,6 +2052,17 @@ static int HandleEstablishedPacketToServer(ThreadVars *tv, TcpSession *ssn, Pack SCLogDebug("ssn %p: ssn->client.next_seq %"PRIu32 " (started before next_seq, ended after)", ssn, ssn->client.next_seq); + /* if next_seq has fallen behind last_ack, we got some catching up to do */ + } else if (SEQ_LT(ssn->client.next_seq, ssn->client.last_ack)) { + ssn->client.next_seq = TCP_GET_SEQ(p) + p->payload_len; + SCLogDebug("ssn %p: ssn->client.next_seq %"PRIu32 + " (next_seq had fallen behind last_ack)", + ssn, ssn->client.next_seq); + } else { + SCLogDebug("ssn %p: no update to ssn->client.next_seq %"PRIu32 + " SEQ %u SEQ+ %u last_ack %u", + ssn, ssn->client.next_seq, + TCP_GET_SEQ(p), TCP_GET_SEQ(p)+p->payload_len, ssn->client.last_ack); } /* in window check */ @@ -2185,6 +2208,17 @@ static int HandleEstablishedPacketToClient(ThreadVars *tv, TcpSession *ssn, Pack SCLogDebug("ssn %p: ssn->server.next_seq %" PRIu32 " (started before next_seq, ended after)", ssn, ssn->server.next_seq); + /* if next_seq has fallen behind last_ack, we got some catching up to do */ + } else if (SEQ_LT(ssn->server.next_seq, ssn->server.last_ack)) { + ssn->server.next_seq = TCP_GET_SEQ(p) + p->payload_len; + SCLogDebug("ssn %p: ssn->server.next_seq %"PRIu32 + " (next_seq had fallen behind last_ack)", + ssn, ssn->server.next_seq); + } else { + SCLogDebug("ssn %p: no update to ssn->server.next_seq %"PRIu32 + " SEQ %u SEQ+ %u last_ack %u", + ssn, ssn->server.next_seq, + TCP_GET_SEQ(p), TCP_GET_SEQ(p)+p->payload_len, ssn->server.last_ack); } if (zerowindowprobe) { @@ -2284,8 +2318,9 @@ static int StreamTcpPacketStateEstablished(ThreadVars *tv, Packet *p, ssn->server.next_seq); ssn->client.window = TCP_GET_WINDOW(p) << ssn->client.wscale; - StreamTcpUpdateLastAck(ssn, &ssn->server, - StreamTcpResetGetMaxAck(&ssn->server, TCP_GET_ACK(p))); + if ((p->tcph->th_flags & TH_ACK) && StreamTcpValidateAck(ssn, &ssn->server, p) == 0) + StreamTcpUpdateLastAck(ssn, &ssn->server, + StreamTcpResetGetMaxAck(&ssn->server, TCP_GET_ACK(p))); StreamTcpUpdateLastAck(ssn, &ssn->client, StreamTcpResetGetMaxAck(&ssn->client, TCP_GET_SEQ(p))); @@ -2315,8 +2350,9 @@ static int StreamTcpPacketStateEstablished(ThreadVars *tv, Packet *p, ssn->server.next_seq); ssn->server.window = TCP_GET_WINDOW(p) << ssn->server.wscale; - StreamTcpUpdateLastAck(ssn, &ssn->client, - StreamTcpResetGetMaxAck(&ssn->client, TCP_GET_ACK(p))); + if ((p->tcph->th_flags & TH_ACK) && StreamTcpValidateAck(ssn, &ssn->client, p) == 0) + StreamTcpUpdateLastAck(ssn, &ssn->client, + StreamTcpResetGetMaxAck(&ssn->client, TCP_GET_ACK(p))); StreamTcpUpdateLastAck(ssn, &ssn->server, StreamTcpResetGetMaxAck(&ssn->server, TCP_GET_SEQ(p))); @@ -2609,8 +2645,9 @@ static int StreamTcpPacketStateFinWait1(ThreadVars *tv, Packet *p, ssn); if (PKT_IS_TOSERVER(p)) { - StreamTcpUpdateLastAck(ssn, &ssn->server, - StreamTcpResetGetMaxAck(&ssn->server, TCP_GET_ACK(p))); + if ((p->tcph->th_flags & TH_ACK) && StreamTcpValidateAck(ssn, &ssn->server, p) == 0) + StreamTcpUpdateLastAck(ssn, &ssn->server, + StreamTcpResetGetMaxAck(&ssn->server, TCP_GET_ACK(p))); StreamTcpUpdateLastAck(ssn, &ssn->client, StreamTcpResetGetMaxAck(&ssn->client, TCP_GET_SEQ(p))); @@ -2622,8 +2659,9 @@ static int StreamTcpPacketStateFinWait1(ThreadVars *tv, Packet *p, StreamTcpReassembleHandleSegment(tv, stt->ra_ctx, ssn, &ssn->client, p, pq); } else { - StreamTcpUpdateLastAck(ssn, &ssn->client, - StreamTcpResetGetMaxAck(&ssn->client, TCP_GET_ACK(p))); + if ((p->tcph->th_flags & TH_ACK) && StreamTcpValidateAck(ssn, &ssn->client, p) == 0) + StreamTcpUpdateLastAck(ssn, &ssn->client, + StreamTcpResetGetMaxAck(&ssn->client, TCP_GET_ACK(p))); StreamTcpUpdateLastAck(ssn, &ssn->server, StreamTcpResetGetMaxAck(&ssn->server, TCP_GET_SEQ(p))); @@ -3063,8 +3101,9 @@ static int StreamTcpPacketStateFinWait2(ThreadVars *tv, Packet *p, ssn); if (PKT_IS_TOSERVER(p)) { - StreamTcpUpdateLastAck(ssn, &ssn->server, - StreamTcpResetGetMaxAck(&ssn->server, TCP_GET_ACK(p))); + if ((p->tcph->th_flags & TH_ACK) && StreamTcpValidateAck(ssn, &ssn->server, p) == 0) + StreamTcpUpdateLastAck(ssn, &ssn->server, + StreamTcpResetGetMaxAck(&ssn->server, TCP_GET_ACK(p))); StreamTcpUpdateLastAck(ssn, &ssn->client, StreamTcpResetGetMaxAck(&ssn->client, TCP_GET_SEQ(p))); @@ -3076,8 +3115,9 @@ static int StreamTcpPacketStateFinWait2(ThreadVars *tv, Packet *p, StreamTcpReassembleHandleSegment(tv, stt->ra_ctx, ssn, &ssn->client, p, pq); } else { - StreamTcpUpdateLastAck(ssn, &ssn->client, - StreamTcpResetGetMaxAck(&ssn->client, TCP_GET_ACK(p))); + if ((p->tcph->th_flags & TH_ACK) && StreamTcpValidateAck(ssn, &ssn->client, p) == 0) + StreamTcpUpdateLastAck(ssn, &ssn->client, + StreamTcpResetGetMaxAck(&ssn->client, TCP_GET_ACK(p))); StreamTcpUpdateLastAck(ssn, &ssn->server, StreamTcpResetGetMaxAck(&ssn->server, TCP_GET_SEQ(p))); @@ -3369,8 +3409,9 @@ static int StreamTcpPacketStateClosing(ThreadVars *tv, Packet *p, ssn); if (PKT_IS_TOSERVER(p)) { - StreamTcpUpdateLastAck(ssn, &ssn->server, - StreamTcpResetGetMaxAck(&ssn->server, TCP_GET_ACK(p))); + if ((p->tcph->th_flags & TH_ACK) && StreamTcpValidateAck(ssn, &ssn->server, p) == 0) + StreamTcpUpdateLastAck(ssn, &ssn->server, + StreamTcpResetGetMaxAck(&ssn->server, TCP_GET_ACK(p))); StreamTcpUpdateLastAck(ssn, &ssn->client, StreamTcpResetGetMaxAck(&ssn->client, TCP_GET_SEQ(p))); @@ -3382,8 +3423,9 @@ static int StreamTcpPacketStateClosing(ThreadVars *tv, Packet *p, StreamTcpReassembleHandleSegment(tv, stt->ra_ctx, ssn, &ssn->client, p, pq); } else { - StreamTcpUpdateLastAck(ssn, &ssn->client, - StreamTcpResetGetMaxAck(&ssn->client, TCP_GET_ACK(p))); + if ((p->tcph->th_flags & TH_ACK) && StreamTcpValidateAck(ssn, &ssn->client, p) == 0) + StreamTcpUpdateLastAck(ssn, &ssn->client, + StreamTcpResetGetMaxAck(&ssn->client, TCP_GET_ACK(p))); StreamTcpUpdateLastAck(ssn, &ssn->server, StreamTcpResetGetMaxAck(&ssn->server, TCP_GET_SEQ(p))); @@ -3549,8 +3591,9 @@ static int StreamTcpPacketStateCloseWait(ThreadVars *tv, Packet *p, ssn); if (PKT_IS_TOSERVER(p)) { - StreamTcpUpdateLastAck(ssn, &ssn->server, - StreamTcpResetGetMaxAck(&ssn->server, TCP_GET_ACK(p))); + if ((p->tcph->th_flags & TH_ACK) && StreamTcpValidateAck(ssn, &ssn->server, p) == 0) + StreamTcpUpdateLastAck(ssn, &ssn->server, + StreamTcpResetGetMaxAck(&ssn->server, TCP_GET_ACK(p))); StreamTcpUpdateLastAck(ssn, &ssn->client, StreamTcpResetGetMaxAck(&ssn->client, TCP_GET_SEQ(p))); @@ -3562,8 +3605,9 @@ static int StreamTcpPacketStateCloseWait(ThreadVars *tv, Packet *p, StreamTcpReassembleHandleSegment(tv, stt->ra_ctx, ssn, &ssn->client, p, pq); } else { - StreamTcpUpdateLastAck(ssn, &ssn->client, - StreamTcpResetGetMaxAck(&ssn->client, TCP_GET_ACK(p))); + if ((p->tcph->th_flags & TH_ACK) && StreamTcpValidateAck(ssn, &ssn->client, p) == 0) + StreamTcpUpdateLastAck(ssn, &ssn->client, + StreamTcpResetGetMaxAck(&ssn->client, TCP_GET_ACK(p))); StreamTcpUpdateLastAck(ssn, &ssn->server, StreamTcpResetGetMaxAck(&ssn->server, TCP_GET_SEQ(p))); @@ -3840,8 +3884,9 @@ static int StreamTcpPacketStateLastAck(ThreadVars *tv, Packet *p, ssn); if (PKT_IS_TOSERVER(p)) { - StreamTcpUpdateLastAck(ssn, &ssn->server, - StreamTcpResetGetMaxAck(&ssn->server, TCP_GET_ACK(p))); + if ((p->tcph->th_flags & TH_ACK) && StreamTcpValidateAck(ssn, &ssn->server, p) == 0) + StreamTcpUpdateLastAck(ssn, &ssn->server, + StreamTcpResetGetMaxAck(&ssn->server, TCP_GET_ACK(p))); StreamTcpUpdateLastAck(ssn, &ssn->client, StreamTcpResetGetMaxAck(&ssn->client, TCP_GET_SEQ(p))); @@ -3853,8 +3898,9 @@ static int StreamTcpPacketStateLastAck(ThreadVars *tv, Packet *p, StreamTcpReassembleHandleSegment(tv, stt->ra_ctx, ssn, &ssn->client, p, pq); } else { - StreamTcpUpdateLastAck(ssn, &ssn->client, - StreamTcpResetGetMaxAck(&ssn->client, TCP_GET_ACK(p))); + if ((p->tcph->th_flags & TH_ACK) && StreamTcpValidateAck(ssn, &ssn->client, p) == 0) + StreamTcpUpdateLastAck(ssn, &ssn->client, + StreamTcpResetGetMaxAck(&ssn->client, TCP_GET_ACK(p))); StreamTcpUpdateLastAck(ssn, &ssn->server, StreamTcpResetGetMaxAck(&ssn->server, TCP_GET_SEQ(p))); @@ -3965,8 +4011,9 @@ static int StreamTcpPacketStateTimeWait(ThreadVars *tv, Packet *p, ssn); if (PKT_IS_TOSERVER(p)) { - StreamTcpUpdateLastAck(ssn, &ssn->server, - StreamTcpResetGetMaxAck(&ssn->server, TCP_GET_ACK(p))); + if ((p->tcph->th_flags & TH_ACK) && StreamTcpValidateAck(ssn, &ssn->server, p) == 0) + StreamTcpUpdateLastAck(ssn, &ssn->server, + StreamTcpResetGetMaxAck(&ssn->server, TCP_GET_ACK(p))); StreamTcpUpdateLastAck(ssn, &ssn->client, StreamTcpResetGetMaxAck(&ssn->client, TCP_GET_SEQ(p))); @@ -3978,8 +4025,9 @@ static int StreamTcpPacketStateTimeWait(ThreadVars *tv, Packet *p, StreamTcpReassembleHandleSegment(tv, stt->ra_ctx, ssn, &ssn->client, p, pq); } else { - StreamTcpUpdateLastAck(ssn, &ssn->client, - StreamTcpResetGetMaxAck(&ssn->client, TCP_GET_ACK(p))); + if ((p->tcph->th_flags & TH_ACK) && StreamTcpValidateAck(ssn, &ssn->client, p) == 0) + StreamTcpUpdateLastAck(ssn, &ssn->client, + StreamTcpResetGetMaxAck(&ssn->client, TCP_GET_ACK(p))); StreamTcpUpdateLastAck(ssn, &ssn->server, StreamTcpResetGetMaxAck(&ssn->server, TCP_GET_SEQ(p))); diff --git a/framework/src/suricata/src/suricata.c b/framework/src/suricata/src/suricata.c index 6c45c57e..173ef75f 100644 --- a/framework/src/suricata/src/suricata.c +++ b/framework/src/suricata/src/suricata.c @@ -1977,14 +1977,20 @@ static int ConfigGetCaptureValue(SCInstance *suri) * back on a sane default. */ char *temp_default_packet_size; if ((ConfGet("default-packet-size", &temp_default_packet_size)) != 1) { + int lthread; + int nlive; switch (suri->run_mode) { case RUNMODE_PCAP_DEV: case RUNMODE_AFP_DEV: case RUNMODE_NETMAP: case RUNMODE_PFRING: - /* FIXME this don't work effficiently in multiinterface */ - /* find payload for interface and use it */ - default_packet_size = GetIfaceMaxPacketSize(suri->pcap_dev); + nlive = LiveGetDeviceCount(); + for (lthread = 0; lthread < nlive; lthread++) { + char *live_dev = LiveGetDeviceName(lthread); + unsigned int iface_max_packet_size = GetIfaceMaxPacketSize(live_dev); + if (iface_max_packet_size > default_packet_size) + default_packet_size = iface_max_packet_size; + } if (default_packet_size) break; /* fall through */ @@ -2226,6 +2232,11 @@ int main(int argc, char **argv) exit(EXIT_FAILURE); } + if (suri.run_mode == RUNMODE_DUMP_CONFIG) { + ConfDump(); + exit(EXIT_SUCCESS); + } + /* Since our config is now loaded we can finish configurating the * logging module. */ SCLogLoadConfig(suri.daemon, suri.verbose); @@ -2234,9 +2245,8 @@ int main(int argc, char **argv) UtilCpuPrintSummary(); - if (suri.run_mode == RUNMODE_DUMP_CONFIG) { - ConfDump(); - exit(EXIT_SUCCESS); + if (ParseInterfacesList(suri.run_mode, suri.pcap_dev) != TM_ECODE_OK) { + exit(EXIT_FAILURE); } if (PostConfLoadedSetup(&suri) != TM_ECODE_OK) { @@ -2320,10 +2330,6 @@ int main(int argc, char **argv) StatsSetupPostConfig(); } - if (ParseInterfacesList(suri.run_mode, suri.pcap_dev) != TM_ECODE_OK) { - exit(EXIT_FAILURE); - } - if(suri.run_mode == RUNMODE_CONF_TEST){ SCLogNotice("Configuration provided was successfully loaded. Exiting."); exit(EXIT_SUCCESS); @@ -2367,6 +2373,7 @@ int main(int argc, char **argv) } (void) SC_ATOMIC_CAS(&engine_stage, SURICATA_INIT, SURICATA_RUNTIME); + PacketPoolPostRunmodes(); /* Un-pause all the paused threads */ TmThreadContinueThreads(); @@ -2379,7 +2386,7 @@ int main(int argc, char **argv) if (suri.delayed_detect) { /* force 'reload', this will load the rules and swap engines */ - DetectEngineReload(NULL); + DetectEngineReload(NULL, &suri); if (suri.sig_file != NULL) UtilSignalHandlerSetup(SIGUSR2, SignalHandlerSigusr2SigFileStartup); @@ -2412,10 +2419,10 @@ int main(int argc, char **argv) } if (sigusr2_count > 0) { - DetectEngineReload(conf_filename); + DetectEngineReload(conf_filename, &suri); sigusr2_count--; } else if (DetectEngineReloadIsStart()) { - DetectEngineReload(conf_filename); + DetectEngineReload(conf_filename, &suri); DetectEngineReloadSetDone(); } diff --git a/framework/src/suricata/src/threadvars.h b/framework/src/suricata/src/threadvars.h index a632a23b..79a2b34d 100644 --- a/framework/src/suricata/src/threadvars.h +++ b/framework/src/suricata/src/threadvars.h @@ -71,6 +71,9 @@ typedef struct ThreadVars_ { /** no of times the thread has been restarted on failure */ uint8_t restarted; + /** TmModule::flags for each module part of this thread */ + uint8_t tmm_flags; + /** local id */ int id; diff --git a/framework/src/suricata/src/tm-threads.c b/framework/src/suricata/src/tm-threads.c index c6828b40..2b26404f 100644 --- a/framework/src/suricata/src/tm-threads.c +++ b/framework/src/suricata/src/tm-threads.c @@ -730,6 +730,7 @@ static inline TmSlot * _TmSlotSetFuncAppend(ThreadVars *tv, TmModule *tm, void * * received a TM as arg, if it didn't exist */ slot->tm_id = TmModuleGetIDForTM(tm); + tv->tmm_flags |= tm->flags; tv->cap_flags |= tm->cap_flags; if (tv->tm_slots == NULL) { @@ -2033,6 +2034,29 @@ ThreadVars *TmThreadsGetCallingThread(void) return NULL; } +/** + * \brief returns a count of all the threads that match the flag + */ +uint32_t TmThreadCountThreadsByTmmFlags(uint8_t flags) +{ + ThreadVars *tv = NULL; + int i = 0; + uint32_t cnt = 0; + + SCMutexLock(&tv_root_lock); + for (i = 0; i < TVT_MAX; i++) { + tv = tv_root[i]; + while (tv != NULL) { + if ((tv->tmm_flags & flags) == flags) + cnt++; + + tv = tv->next; + } + } + SCMutexUnlock(&tv_root_lock); + return cnt; +} + typedef struct Thread_ { ThreadVars *tv; /**< threadvars structure */ const char *name; diff --git a/framework/src/suricata/src/tm-threads.h b/framework/src/suricata/src/tm-threads.h index 9b9fd620..397fbd2e 100644 --- a/framework/src/suricata/src/tm-threads.h +++ b/framework/src/suricata/src/tm-threads.h @@ -132,6 +132,8 @@ void TmThreadDisablePacketThreads(void); void TmThreadDisableReceiveThreads(void); TmSlot *TmThreadGetFirstTmSlotForPartialPattern(const char *); +uint32_t TmThreadCountThreadsByTmmFlags(uint8_t flags); + /** * \brief Process the rest of the functions (if any) and queue. */ diff --git a/framework/src/suricata/src/tmqh-packetpool.c b/framework/src/suricata/src/tmqh-packetpool.c index a1f19dca..75139254 100644 --- a/framework/src/suricata/src/tmqh-packetpool.c +++ b/framework/src/suricata/src/tmqh-packetpool.c @@ -38,6 +38,8 @@ #include "stream-tcp-reassemble.h" #include "tm-queuehandlers.h" +#include "tm-threads.h" +#include "tm-modules.h" #include "pkt-var.h" @@ -50,6 +52,7 @@ /* Number of freed packet to save for one pool before freeing them. */ #define MAX_PENDING_RETURN_PACKETS 32 +static uint32_t max_pending_return_packets = MAX_PENDING_RETURN_PACKETS; #ifdef TLS __thread PktPool thread_pkt_pool; @@ -313,7 +316,7 @@ void PacketPoolReturnPacket(Packet *p) p->next = my_pool->pending_head; my_pool->pending_head = p; my_pool->pending_count++; - if (SC_ATOMIC_GET(pool->return_stack.sync_now) || my_pool->pending_count > MAX_PENDING_RETURN_PACKETS) { + if (SC_ATOMIC_GET(pool->return_stack.sync_now) || my_pool->pending_count > max_pending_return_packets) { /* Return the entire list of pending packets. */ SCMutexLock(&pool->return_stack.mutex); my_pool->pending_tail->next = pool->return_stack.head; @@ -563,3 +566,34 @@ void TmqhReleasePacketsToPacketPool(PacketQueue *pq) return; } + +/** + * \brief Set the max_pending_return_packets value + * + * Set it to the max pending packets value, devided by the number + * of lister threads. Normally, in autofp these are the stream/detect/log + * worker threads. + * + * The max_pending_return_packets value needs to stay below the packet + * pool size of the 'producers' (normally pkt capture threads but also + * flow timeout injection ) to avoid a deadlock where all the 'workers' + * keep packets in their return pools, while the capture thread can't + * continue because its pool is empty. + */ +void PacketPoolPostRunmodes(void) +{ + extern intmax_t max_pending_packets; + + uint32_t threads = TmThreadCountThreadsByTmmFlags(TM_FLAG_DETECT_TM); + if (threads == 0) + return; + if (threads > max_pending_packets) + return; + + uint32_t packets = (max_pending_packets / threads) - 1; + if (packets < max_pending_return_packets) + max_pending_return_packets = packets; + + SCLogDebug("detect threads %u, max packets %u, max_pending_return_packets %u", + threads, (uint)threads, max_pending_return_packets); +} diff --git a/framework/src/suricata/src/tmqh-packetpool.h b/framework/src/suricata/src/tmqh-packetpool.h index ab45184c..2b6b90b0 100644 --- a/framework/src/suricata/src/tmqh-packetpool.h +++ b/framework/src/suricata/src/tmqh-packetpool.h @@ -78,5 +78,6 @@ void PacketPoolReturnPacket(Packet *p); void PacketPoolInit(void); void PacketPoolInitEmpty(void); void PacketPoolDestroy(void); +void PacketPoolPostRunmodes(void); #endif /* __TMQH_PACKETPOOL_H__ */ diff --git a/framework/src/suricata/src/unix-manager.c b/framework/src/suricata/src/unix-manager.c index 1960df56..9357f4c5 100644 --- a/framework/src/suricata/src/unix-manager.c +++ b/framework/src/suricata/src/unix-manager.c @@ -356,7 +356,7 @@ int UnixCommandAccept(UnixCommand *this) json_decref(server_msg); /* client connected */ - SCLogInfo("Unix socket: client connected"); + SCLogDebug("Unix socket: client connected"); uclient = SCMalloc(sizeof(UnixClient)); if (unlikely(uclient == NULL)) { @@ -478,9 +478,9 @@ void UnixCommandRun(UnixCommand * this, UnixClient *client) ret = recv(client->fd, buffer, sizeof(buffer) - 1, 0); if (ret <= 0) { if (ret == 0) { - SCLogInfo("Unix socket: lost connection with client"); + SCLogDebug("Unix socket: lost connection with client"); } else { - SCLogInfo("Unix socket: error on recv() from client: %s", + SCLogError(SC_ERR_SOCKET, "Unix socket: error on recv() from client: %s", strerror(errno)); } UnixCommandClose(this, client->fd); @@ -525,7 +525,7 @@ int UnixMain(UnixCommand * this) if (errno == EINTR) { return 1; } - SCLogInfo("Command server: select() fatal error: %s", strerror(errno)); + SCLogError(SC_ERR_SOCKET, "Command server: select() fatal error: %s", strerror(errno)); return 0; } diff --git a/framework/src/suricata/src/util-debug.c b/framework/src/suricata/src/util-debug.c index f1ac0463..d9af08c4 100644 --- a/framework/src/suricata/src/util-debug.c +++ b/framework/src/suricata/src/util-debug.c @@ -776,7 +776,7 @@ static inline SCLogOPIfaceCtx *SCLogInitConsoleOPIface(const char *log_format, } iface_ctx->log_level = tmp_log_level; - if (isatty(fileno(stdout))) { + if (isatty(fileno(stdout)) && isatty(fileno(stderr))) { iface_ctx->use_color = TRUE; } diff --git a/framework/src/suricata/src/util-decode-mime.c b/framework/src/suricata/src/util-decode-mime.c index 3f4affcc..51d1468e 100644 --- a/framework/src/suricata/src/util-decode-mime.c +++ b/framework/src/suricata/src/util-decode-mime.c @@ -79,9 +79,8 @@ #define MAX_IP6_CHARS 39 /* Globally hold configuration data */ -static MimeDecConfig mime_dec_config = { 1, 1, 1, MAX_HEADER_VALUE }; +static MimeDecConfig mime_dec_config = { 1, 1, 1, 0, MAX_HEADER_VALUE }; -#ifdef DEBUG /* Mime Parser String translation */ static const char *StateFlags[] = { "NONE", "HEADER_READY", @@ -93,7 +92,6 @@ static const char *StateFlags[] = { "NONE", "PARSE_DONE", "PARSE_ERROR", NULL }; -#endif /* URL executable file extensions */ static const char *UrlExeExts[] = { ".exe", @@ -299,6 +297,35 @@ MimeDecField * MimeDecAddField(MimeDecEntity *entity) return node; } + +/** + * \brief Searches for header fields with the specified name + * + * \param entity The entity to search + * \param name The header name (lowercase) + * + * \return number of items found + * + */ +int MimeDecFindFieldsForEach(const MimeDecEntity *entity, const char *name, int (*DataCallback)(const uint8_t *val, const size_t, void *data), void *data) +{ + MimeDecField *curr = entity->field_list; + int found = 0; + + while (curr != NULL) { + /* name is stored lowercase */ + if (strlen(name) == curr->name_len) { + if (SCMemcmp(curr->name, name, curr->name_len) == 0) { + if (DataCallback(curr->value, curr->value_len, data)) + found++; + } + } + curr = curr->next; + } + + return found; +} + /** * \brief Searches for a header field with the specified name * @@ -915,7 +942,7 @@ static int IsIpv4Host(const uint8_t *urlhost, uint32_t len) */ static int IsIpv6Host(const uint8_t *urlhost, uint32_t len) { - struct sockaddr_in sa; + struct in6_addr in6; char tempIp[MAX_IP6_CHARS + 1]; /* Cut off at '/' */ @@ -936,7 +963,7 @@ static int IsIpv6Host(const uint8_t *urlhost, uint32_t len) memcpy(tempIp, urlhost, i); tempIp[i] = '\0'; - return inet_pton(AF_INET6, tempIp, &(sa.sin_addr)); + return inet_pton(AF_INET6, tempIp, &in6); } /** @@ -1992,6 +2019,7 @@ static int ProcessMimeHeaders(const uint8_t *buf, uint32_t len, * * \return MIME_DEC_OK on success, otherwise < 0 on failure */ + static int ProcessBodyComplete(MimeDecParseState *state) { int ret = MIME_DEC_OK; @@ -2012,6 +2040,13 @@ static int ProcessBodyComplete(MimeDecParseState *state) } } +#ifdef HAVE_NSS + if (state->md5_ctx) { + unsigned int len = 0; + HASH_End(state->md5_ctx, state->md5, &len, sizeof(state->md5)); + } +#endif + /* Invoke pre-processor and callback with remaining data */ ret = ProcessDecodedDataChunk(state->data_chunk, state->data_chunk_len, state); if (ret != MIME_DEC_OK) { @@ -2180,6 +2215,18 @@ static int ProcessMimeBody(const uint8_t *buf, uint32_t len, int body_found = 0; uint32_t tlen; +#ifdef HAVE_NSS + if (MimeDecGetConfig()->body_md5) { + if (state->body_begin == 1) { + if (state->md5_ctx == NULL) { + state->md5_ctx = HASH_Create(HASH_AlgMD5); + HASH_Begin(state->md5_ctx); + } + } + HASH_Update(state->md5_ctx, buf, len + state->current_line_delimiter_len); + } +#endif + /* Ignore empty lines */ if (len == 0) { return ret; @@ -2254,6 +2301,11 @@ static int ProcessMimeBody(const uint8_t *buf, uint32_t len, return ret; } +const char *MimeDecParseStateGetStatus(MimeDecParseState *state) +{ + return StateFlags[state->state_flag]; +} + /** * \brief Processes the MIME Entity based on the input line and current state of * the parser @@ -2394,6 +2446,10 @@ void MimeDecDeInitParser(MimeDecParseState *state) SCFree(state->hname); FreeDataValue(state->hvalue); FreeMimeDecStack(state->stack); +#ifdef HAVE_NSS + if (state->md5_ctx) + HASH_Destroy(state->md5_ctx); +#endif SCFree(state); } @@ -2459,12 +2515,13 @@ int MimeDecParseComplete(MimeDecParseState *state) * * \param line A string representing the line (w/out CRLF) * \param len The length of the line + * \param delim_len The length of the line end delimiter * \param state The parser state * * \return MIME_DEC_OK on success, otherwise < 0 on failure */ int MimeDecParseLine(const uint8_t *line, const uint32_t len, - MimeDecParseState *state) + const uint8_t delim_len, MimeDecParseState *state) { int ret = MIME_DEC_OK; @@ -2475,6 +2532,7 @@ int MimeDecParseLine(const uint8_t *line, const uint32_t len, SCLogDebug("SMTP LINE - EMPTY"); } + state->current_line_delimiter_len = delim_len; /* Process the entity */ ret = ProcessMimeEntity(line, len, state); if (ret != MIME_DEC_OK) { @@ -2522,8 +2580,10 @@ MimeDecEntity * MimeDecParseFullMsg(const uint8_t *buf, uint32_t blen, void *dat line = tok; + state->current_line_delimiter_len = (remainPtr - tok) - tokLen; /* Parse the line */ - ret = MimeDecParseLine(line, tokLen, state); + ret = MimeDecParseLine(line, tokLen, + (remainPtr - tok) - tokLen, state); if (ret != MIME_DEC_OK) { SCLogDebug("Error: MimeDecParseLine() function failed: %d", ret); @@ -2610,25 +2670,25 @@ static int MimeDecParseLineTest01(void) TestDataChunkCallback); char *str = "From: Sender1"; - ret |= MimeDecParseLine((uint8_t *)str, strlen(str), state); + ret |= MimeDecParseLine((uint8_t *)str, strlen(str), 1, state); str = "To: Recipient1"; - ret |= MimeDecParseLine((uint8_t *)str, strlen(str), state); + ret |= MimeDecParseLine((uint8_t *)str, strlen(str), 1, state); str = "Content-Type: text/plain"; - ret |= MimeDecParseLine((uint8_t *)str, strlen(str), state); + ret |= MimeDecParseLine((uint8_t *)str, strlen(str), 1, state); str = ""; - ret |= MimeDecParseLine((uint8_t *)str, strlen(str), state); + ret |= MimeDecParseLine((uint8_t *)str, strlen(str), 1, state); str = "A simple message line 1"; - ret |= MimeDecParseLine((uint8_t *)str, strlen(str), state); + ret |= MimeDecParseLine((uint8_t *)str, strlen(str), 1, state); str = "A simple message line 2"; - ret |= MimeDecParseLine((uint8_t *)str, strlen(str), state); + ret |= MimeDecParseLine((uint8_t *)str, strlen(str), 1, state); str = "A simple message line 3"; - ret |= MimeDecParseLine((uint8_t *)str, strlen(str), state); + ret |= MimeDecParseLine((uint8_t *)str, strlen(str), 1, state); if (ret != MIME_DEC_OK) { return ret; @@ -2678,23 +2738,23 @@ static int MimeDecParseLineTest02(void) TestDataChunkCallback); char *str = "From: Sender1"; - ret |= MimeDecParseLine((uint8_t *)str, strlen(str), state); + ret |= MimeDecParseLine((uint8_t *)str, strlen(str), 1, state); str = "To: Recipient1"; - ret |= MimeDecParseLine((uint8_t *)str, strlen(str), state); + ret |= MimeDecParseLine((uint8_t *)str, strlen(str), 1, state); str = "Content-Type: text/plain"; - ret |= MimeDecParseLine((uint8_t *)str, strlen(str), state); + ret |= MimeDecParseLine((uint8_t *)str, strlen(str), 1, state); str = ""; - ret |= MimeDecParseLine((uint8_t *)str, strlen(str), state); + ret |= MimeDecParseLine((uint8_t *)str, strlen(str), 1, state); str = "A simple message line 1"; - ret |= MimeDecParseLine((uint8_t *)str, strlen(str), state); + ret |= MimeDecParseLine((uint8_t *)str, strlen(str), 1, state); str = "A simple message line 2 click on http://www.test.com/malware.exe?" "hahah hopefully you click this link"; - ret |= MimeDecParseLine((uint8_t *)str, strlen(str), state); + ret |= MimeDecParseLine((uint8_t *)str, strlen(str), 1, state); if (ret != MIME_DEC_OK) { return ret; @@ -2762,6 +2822,59 @@ static int MimeDecParseFullMsgTest01(void) return ret; } +/* Test full message with linebreaks */ +static int MimeDecParseFullMsgTest02(void) +{ + int ret = MIME_DEC_OK; + + uint32_t expected_count = 3; + uint32_t line_count = 0; + + char msg[] = "From: Sender2\r\n" + "To: Recipient2\r\n" + "Subject: subject2\r\n" + "Content-Type: text/plain\r\n" + "\r\n" + "Line 1\r\n" + "Line 2\r\n" + "Line 3\r\n"; + + MimeDecEntity *entity = MimeDecParseFullMsg((uint8_t *)msg, strlen(msg), &line_count, + TestDataChunkCallback); + + if (entity == NULL) { + SCLogInfo("Warning: Message failed to parse"); + return -1; + } + + MimeDecField *field = MimeDecFindField(entity, "subject"); + if (field == NULL) { + SCLogInfo("Warning: Message failed to parse"); + return -1; + } + + if (field->value_len != sizeof("subject2") - 1) { + SCLogInfo("Warning: failed to get subject"); + return -1; + } + + if (memcmp(field->value, "subject2", field->value_len) != 0) { + SCLogInfo("Warning: failed to get subject"); + return -1; + } + + + MimeDecFreeEntity(entity); + + if (expected_count != line_count) { + SCLogInfo("Warning: Line count is invalid: expected - %d actual - %d", + expected_count, line_count); + return -1; + } + + return ret; +} + static int MimeBase64DecodeTest01(void) { int ret = -1; @@ -2771,7 +2884,7 @@ static int MimeBase64DecodeTest01(void) char *base64msg = "YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUZHSElKS0xNTk9QU" "VJTVFVWV1hZWjEyMzQ1Njc4OTBAIyQlXiYqKCktPV8rLC4vOydbXTw+Pzo="; - uint8_t *dst = SCMalloc(strlen(msg)-1); + uint8_t *dst = SCMalloc(strlen(msg) + 1); if (dst == NULL) return 0; @@ -2781,6 +2894,8 @@ static int MimeBase64DecodeTest01(void) ret = 0; } + SCFree(dst); + return ret; } @@ -2879,6 +2994,7 @@ void MimeDecRegisterTests(void) UtRegisterTest("MimeDecParseLineTest01", MimeDecParseLineTest01, 0); UtRegisterTest("MimeDecParseLineTest02", MimeDecParseLineTest02, 0); UtRegisterTest("MimeDecParseFullMsgTest01", MimeDecParseFullMsgTest01, 0); + UtRegisterTest("MimeDecParseFullMsgTest02", MimeDecParseFullMsgTest02, 0); UtRegisterTest("MimeBase64DecodeTest01", MimeBase64DecodeTest01, 0); UtRegisterTest("MimeIsExeURLTest01", MimeIsExeURLTest01, 0); UtRegisterTest("MimeIsIpv4HostTest01", MimeIsIpv4HostTest01, 0); diff --git a/framework/src/suricata/src/util-decode-mime.h b/framework/src/suricata/src/util-decode-mime.h index 792fbcc1..02b3bb13 100644 --- a/framework/src/suricata/src/util-decode-mime.h +++ b/framework/src/suricata/src/util-decode-mime.h @@ -33,9 +33,6 @@ #include "util-base64.h" #include "util-debug.h" -/* Header Flags */ -#define HDR_IS_LOGGED 1 - /* Content Flags */ #define CTNT_IS_MSG 1 #define CTNT_IS_ENV 2 @@ -64,7 +61,7 @@ #define ANOM_MALFORMED_MSG 64 /* Misc msg format errors found */ #define ANOM_LONG_BOUNDARY 128 /* Boundary too long */ -/* Pubicly exposed size constants */ +/* Publicly exposed size constants */ #define DATA_CHUNK_SIZE 3072 /* Should be divisible by 3 */ #define LINEREM_SIZE 256 @@ -97,6 +94,7 @@ typedef struct MimeDecConfig { int decode_base64; /**< Decode base64 bodies */ int decode_quoted_printable; /**< Decode quoted-printable bodies */ int extract_urls; /**< Extract and store URLs in data structure */ + int body_md5; /**< Compute md5 sum of body */ uint32_t header_value_depth; /**< Depth of which to store header values (Default is 2000) */ } MimeDecConfig; @@ -197,11 +195,16 @@ typedef struct MimeDecParseState { uint8_t bvremain[B64_BLOCK]; /**< Remainder from base64-decoded line */ uint8_t bvr_len; /**< Length of remainder from base64-decoded line */ uint8_t data_chunk[DATA_CHUNK_SIZE]; /**< Buffer holding data chunk */ +#ifdef HAVE_NSS + HASHContext *md5_ctx; + uint8_t md5[MD5_LENGTH]; +#endif uint8_t state_flag; /**< Flag representing current state of parser */ uint32_t data_chunk_len; /**< Length of data chunk */ int found_child; /**< Flag indicating a child entity was found */ int body_begin; /**< Currently at beginning of body */ int body_end; /**< Currently at end of body */ + uint8_t current_line_delimiter_len; /**< Length of line delimiter */ void *data; /**< Pointer to data specific to the caller */ int (*DataChunkProcessorFunc) (const uint8_t *chunk, uint32_t len, struct MimeDecParseState *state); /**< Data chunk processing function callback */ @@ -219,6 +222,7 @@ void MimeDecFreeUrl(MimeDecUrl *url); /* List functions */ MimeDecField * MimeDecAddField(MimeDecEntity *entity); MimeDecField * MimeDecFindField(const MimeDecEntity *entity, const char *name); +int MimeDecFindFieldsForEach(const MimeDecEntity *entity, const char *name, int (*DataCallback)(const uint8_t *val, const size_t, void *data), void *data); MimeDecEntity * MimeDecAddEntity(MimeDecEntity *parent); /* Helper functions */ @@ -230,9 +234,10 @@ MimeDecParseState * MimeDecInitParser(void *data, int (*dcpfunc)(const uint8_t * uint32_t len, MimeDecParseState *state)); void MimeDecDeInitParser(MimeDecParseState *state); int MimeDecParseComplete(MimeDecParseState *state); -int MimeDecParseLine(const uint8_t *line, const uint32_t len, MimeDecParseState *state); +int MimeDecParseLine(const uint8_t *line, const uint32_t len, const uint8_t delim_len, MimeDecParseState *state); MimeDecEntity * MimeDecParseFullMsg(const uint8_t *buf, uint32_t blen, void *data, int (*DataChunkProcessorFunc)(const uint8_t *chunk, uint32_t len, MimeDecParseState *state)); +const char *MimeDecParseStateGetStatus(MimeDecParseState *state); /* Test functions */ void MimeDecRegisterTests(void); diff --git a/framework/src/suricata/src/util-error.c b/framework/src/suricata/src/util-error.c index 6e783f42..461e1870 100644 --- a/framework/src/suricata/src/util-error.c +++ b/framework/src/suricata/src/util-error.c @@ -309,6 +309,7 @@ const char * SCErrorToString(SCError err) CASE_CODE (SC_ERR_IPPAIR_INIT); CASE_CODE (SC_ERR_MT_NO_SELECTOR); CASE_CODE (SC_ERR_MT_DUPLICATE_TENANT); + CASE_CODE (SC_ERR_NO_JSON_SUPPORT); } return "UNKNOWN_ERROR"; diff --git a/framework/src/suricata/src/util-error.h b/framework/src/suricata/src/util-error.h index d1e42a8c..b7df4766 100644 --- a/framework/src/suricata/src/util-error.h +++ b/framework/src/suricata/src/util-error.h @@ -298,6 +298,7 @@ typedef enum { SC_ERR_IPPAIR_INIT, SC_ERR_MT_NO_SELECTOR, SC_ERR_MT_DUPLICATE_TENANT, + SC_ERR_NO_JSON_SUPPORT, } SCError; const char *SCErrorToString(SCError); diff --git a/framework/src/suricata/src/util-host-os-info.c b/framework/src/suricata/src/util-host-os-info.c index ad0de7e2..f04d6326 100644 --- a/framework/src/suricata/src/util-host-os-info.c +++ b/framework/src/suricata/src/util-host-os-info.c @@ -450,12 +450,12 @@ int SCHInfoTestInvalidIPV4Address02(void) { SCHInfoCreateContextBackup(); - int result = 1; + int result = 0; if (SCHInfoAddHostOSInfo("linux", "192.168.1.566", SC_HINFO_IS_IPV4) != -1) { goto end; } - if (SCHInfoAddHostOSInfo("linux", "192.168.1", SC_HINFO_IS_IPV4 != -1)) { + if (SCHInfoAddHostOSInfo("linux", "192.168.1", SC_HINFO_IS_IPV4) != -1) { goto end; } if (SCHInfoAddHostOSInfo("linux", "192.", SC_HINFO_IS_IPV4) != -1) { @@ -488,7 +488,7 @@ int SCHInfoTestInvalidIPV6Address03(void) { SCHInfoCreateContextBackup(); - int result = 1; + int result = 0; if (SCHInfoAddHostOSInfo("linux", "2362:7322", SC_HINFO_IS_IPV6) != -1) { goto end; @@ -530,7 +530,7 @@ int SCHInfoTestValidIPV4Address04(void) { SCHInfoCreateContextBackup(); - int result = 1; + int result = 0; if (SCHInfoAddHostOSInfo("linux", "192.168.1.1", SC_HINFO_IS_IPV4) == -1) { goto end; @@ -643,7 +643,7 @@ int SCHInfoTestValidIPV4Address05(void) SCHInfoCreateContextBackup(); struct in_addr in; - int result = 1; + int result = 0; if (SCHInfoAddHostOSInfo("linux", "192.168.1.1", SC_HINFO_IS_IPV4) == -1) { goto end; @@ -783,7 +783,7 @@ int SCHInfoTestValidIPV6Address06(void) { SCHInfoCreateContextBackup(); - int result = 1; + int result = 0; if (SCHInfoAddHostOSInfo("linux", "2351:2512:6211:6246:235A:6242:2352:62AD", @@ -919,7 +919,7 @@ int SCHInfoTestValidIPV6Address07(void) { SCHInfoCreateContextBackup(); - int result = 1; + int result = 0; if (SCHInfoAddHostOSInfo("linux", "2351:2512:6211:6246:235A:6242:2352:62AD", @@ -1076,7 +1076,7 @@ int SCHInfoTestValidIPV6Address08(void) SCHInfoCreateContextBackup(); struct in6_addr in6; - int result = 1; + int result = 0; if (SCHInfoAddHostOSInfo("linux", "2351:2512:6211:6246:235A:6242:2352:62AD", @@ -1248,10 +1248,10 @@ int SCHInfoTestValidIPV4Address09(void) { SCHInfoCreateContextBackup(); - int result = 1; + int result = 0; if (SCHInfoAddHostOSInfo("linux", "192.168.1.0", SC_HINFO_IS_IPV4) == -1) { - goto end; + goto end; } if (SCHInfoAddHostOSInfo("windows", "192.192.1.2", SC_HINFO_IS_IPV4) == -1) { goto end; @@ -1319,22 +1319,26 @@ int SCHInfoTestValidIPV4Address09(void) goto end; } - if (SCHInfoGetHostOSFlavour("192.168.1.100") == + /* 192.168.1.100 should match "macos" as its more specific than + * "solaris". */ + if (SCHInfoGetHostOSFlavour("192.168.1.100") != SCMapEnumNameToValue("macos", sc_hinfo_os_policy_map)) { goto end; } + /* Remove the 192.168.1.0/20 -> macos entry. */ bzero(&servaddr, sizeof(servaddr)); if (inet_pton(AF_INET, "192.168.0.0", &servaddr.sin_addr) <= 0) { goto end; } SCRadixRemoveKeyIPV4Netblock((uint8_t *)&servaddr.sin_addr, sc_hinfo_tree, 20); - if (SCHInfoGetHostOSFlavour("192.168.1.100") == + if (SCHInfoGetHostOSFlavour("192.168.1.100") != SCMapEnumNameToValue("solaris", sc_hinfo_os_policy_map)) { goto end; } + /* Remove the 192.168.1.0/16 -> solaris entry. */ bzero(&servaddr, sizeof(servaddr)); if (inet_pton(AF_INET, "192.168.0.0", &servaddr.sin_addr) <= 0) { goto end; @@ -1481,7 +1485,7 @@ host-os-policy:\n\ vista: [0.0.0.5]\n\ \n"; - int result = 1; + int result = 0; SCHInfoCreateContextBackup(); @@ -1527,7 +1531,7 @@ host-os-policy:\n\ vista: [0.0.0.5]\n\ \n"; - int result = 1; + int result = 0; SCHInfoCreateContextBackup(); diff --git a/framework/src/suricata/src/util-logopenfile.c b/framework/src/suricata/src/util-logopenfile.c index b25c4a82..65b80fac 100644 --- a/framework/src/suricata/src/util-logopenfile.c +++ b/framework/src/suricata/src/util-logopenfile.c @@ -34,6 +34,9 @@ #include "util-logopenfile.h" #include "util-logopenfile-tile.h" +const char * redis_push_cmd = "LPUSH"; +const char * redis_publish_cmd = "PUBLISH"; + /** \brief connect to the indicated local stream socket, logging any errors * \param path filesystem path to connect to * \param log_err, non-zero if connect failure should be logged. @@ -330,6 +333,138 @@ int SCConfLogReopen(LogFileCtx *log_ctx) return 0; } + +#ifdef HAVE_LIBHIREDIS + +static void SCLogFileCloseRedis(LogFileCtx *log_ctx) +{ + if (log_ctx->redis) { + redisReply *reply; + int i; + for (i = 0; i < log_ctx->redis_setup.batch_count; i++) { + redisGetReply(log_ctx->redis, (void **)&reply); + if (reply) + freeReplyObject(reply); + } + redisFree(log_ctx->redis); + log_ctx->redis = NULL; + } + log_ctx->redis_setup.tried = 0; + log_ctx->redis_setup.batch_count = 0; +} + +int SCConfLogOpenRedis(ConfNode *redis_node, LogFileCtx *log_ctx) +{ + const char *redis_server = NULL; + const char *redis_port = NULL; + const char *redis_mode = NULL; + const char *redis_key = NULL; + + if (redis_node) { + redis_server = ConfNodeLookupChildValue(redis_node, "server"); + redis_port = ConfNodeLookupChildValue(redis_node, "port"); + redis_mode = ConfNodeLookupChildValue(redis_node, "mode"); + redis_key = ConfNodeLookupChildValue(redis_node, "key"); + } + if (!redis_server) { + redis_server = "127.0.0.1"; + SCLogInfo("Using default redis server (127.0.0.1)"); + } + if (!redis_port) + redis_port = "6379"; + if (!redis_mode) + redis_mode = "list"; + if (!redis_key) + redis_key = "suricata"; + log_ctx->redis_setup.key = SCStrdup(redis_key); + + if (!log_ctx->redis_setup.key) { + SCLogError(SC_ERR_MEM_ALLOC, "Unable to allocate redis key name"); + exit(EXIT_FAILURE); + } + + log_ctx->redis_setup.batch_size = 0; + + ConfNode *pipelining = ConfNodeLookupChild(redis_node, "pipelining"); + if (pipelining) { + int enabled = 0; + int ret; + intmax_t val; + ret = ConfGetChildValueBool(pipelining, "enabled", &enabled); + if (ret && enabled) { + ret = ConfGetChildValueInt(pipelining, "batch-size", &val); + if (ret) { + log_ctx->redis_setup.batch_size = val; + } else { + log_ctx->redis_setup.batch_size = 10; + } + } + } + + if (!strcmp(redis_mode, "list")) { + log_ctx->redis_setup.command = redis_push_cmd; + if (!log_ctx->redis_setup.command) { + SCLogError(SC_ERR_MEM_ALLOC, "Unable to allocate redis key command"); + exit(EXIT_FAILURE); + } + } else { + log_ctx->redis_setup.command = redis_publish_cmd; + if (!log_ctx->redis_setup.command) { + SCLogError(SC_ERR_MEM_ALLOC, "Unable to allocate redis key command"); + exit(EXIT_FAILURE); + } + } + redisContext *c = redisConnect(redis_server, atoi(redis_port)); + if (c != NULL && c->err) { + SCLogError(SC_ERR_SOCKET, "Error connecting to redis server: %s", c->errstr); + exit(EXIT_FAILURE); + } + + /* store server params for reconnection */ + log_ctx->redis_setup.server = SCStrdup(redis_server); + if (!log_ctx->redis_setup.server) { + SCLogError(SC_ERR_MEM_ALLOC, "Error allocating redis server string"); + exit(EXIT_FAILURE); + } + log_ctx->redis_setup.port = atoi(redis_port); + log_ctx->redis_setup.tried = 0; + + log_ctx->redis = c; + + log_ctx->Close = SCLogFileCloseRedis; + + return 0; +} + +int SCConfLogReopenRedis(LogFileCtx *log_ctx) +{ + if (log_ctx->redis != NULL) { + redisFree(log_ctx->redis); + log_ctx->redis = NULL; + } + + /* only try to reconnect once per second */ + if (log_ctx->redis_setup.tried >= time(NULL)) { + return -1; + } + + redisContext *c = redisConnect(log_ctx->redis_setup.server, log_ctx->redis_setup.port); + if (c != NULL && c->err) { + if (log_ctx->redis_setup.tried == 0) { + SCLogError(SC_ERR_SOCKET, "Error connecting to redis server: %s\n", c->errstr); + } + redisFree(c); + log_ctx->redis_setup.tried = time(NULL); + return -1; + } + log_ctx->redis = c; + log_ctx->redis_setup.tried = 0; + log_ctx->redis_setup.batch_count = 0; + return 0; +} + +#endif + /** \brief LogFileNewCtx() Get a new LogFileCtx * \retval LogFileCtx * pointer if succesful, NULL if error * */ @@ -348,6 +483,10 @@ LogFileCtx *LogFileNewCtx(void) lf_ctx->Write = SCLogFileWrite; lf_ctx->Close = SCLogFileClose; +#ifdef HAVE_LIBHIREDIS + lf_ctx->redis_setup.batch_count = 0; +#endif + return lf_ctx; } @@ -367,6 +506,17 @@ int LogFileFreeCtx(LogFileCtx *lf_ctx) SCMutexUnlock(&lf_ctx->fp_mutex); } +#ifdef HAVE_LIBHIREDIS + if (lf_ctx->type == LOGFILE_TYPE_REDIS) { + if (lf_ctx->redis) + redisFree(lf_ctx->redis); + if (lf_ctx->redis_setup.server) + SCFree(lf_ctx->redis_setup.server); + if (lf_ctx->redis_setup.key) + SCFree(lf_ctx->redis_setup.key); + } +#endif + SCMutexDestroy(&lf_ctx->fp_mutex); if (lf_ctx->prefix != NULL) @@ -375,9 +525,120 @@ int LogFileFreeCtx(LogFileCtx *lf_ctx) if(lf_ctx->filename != NULL) SCFree(lf_ctx->filename); + if (lf_ctx->sensor_name) + SCFree(lf_ctx->sensor_name); + OutputUnregisterFileRotationFlag(&lf_ctx->rotation_flag); SCFree(lf_ctx); SCReturnInt(1); } + +#ifdef HAVE_LIBHIREDIS +static int LogFileWriteRedis(LogFileCtx *file_ctx, char *string, size_t string_len) +{ + if (file_ctx->redis == NULL) { + SCConfLogReopenRedis(file_ctx); + if (file_ctx->redis == NULL) { + return -1; + } else { + SCLogInfo("Reconnected to redis server"); + } + } + /* TODO go async here ? */ + if (file_ctx->redis_setup.batch_size) { + redisAppendCommand(file_ctx->redis, "%s %s %s", + file_ctx->redis_setup.command, + file_ctx->redis_setup.key, + string); + if (file_ctx->redis_setup.batch_count == file_ctx->redis_setup.batch_size) { + redisReply *reply; + int i; + file_ctx->redis_setup.batch_count = 0; + for (i = 0; i <= file_ctx->redis_setup.batch_size; i++) { + if (redisGetReply(file_ctx->redis, (void **)&reply) == REDIS_OK) { + freeReplyObject(reply); + } else { + if (file_ctx->redis->err) { + SCLogInfo("Error when fetching reply: %s (%d)", + file_ctx->redis->errstr, + file_ctx->redis->err); + } + switch (file_ctx->redis->err) { + case REDIS_ERR_EOF: + case REDIS_ERR_IO: + SCLogInfo("Reopening connection to redis server"); + SCConfLogReopenRedis(file_ctx); + if (file_ctx->redis) { + SCLogInfo("Reconnected to redis server"); + return 0; + } else { + SCLogInfo("Unable to reconnect to redis server"); + return 0; + } + break; + default: + SCLogWarning(SC_ERR_INVALID_VALUE, + "Unsupported error code %d", + file_ctx->redis->err); + return 0; + } + } + } + } else { + file_ctx->redis_setup.batch_count++; + } + } else { + redisReply *reply = redisCommand(file_ctx->redis, "%s %s %s", + file_ctx->redis_setup.command, + file_ctx->redis_setup.key, + string); + + switch (reply->type) { + case REDIS_REPLY_ERROR: + SCLogWarning(SC_ERR_SOCKET, "Redis error: %s", reply->str); + SCConfLogReopenRedis(file_ctx); + break; + case REDIS_REPLY_INTEGER: + SCLogDebug("Redis integer %lld", reply->integer); + break; + default: + SCLogError(SC_ERR_INVALID_VALUE, + "Redis default triggered with %d", reply->type); + SCConfLogReopenRedis(file_ctx); + break; + } + freeReplyObject(reply); + } + return 0; +} +#endif + +int LogFileWrite(LogFileCtx *file_ctx, MemBuffer *buffer) +{ + if (file_ctx->type == LOGFILE_TYPE_SYSLOG) { + syslog(file_ctx->syslog_setup.alert_syslog_level, "%s", + (const char *)MEMBUFFER_BUFFER(buffer)); + } else if (file_ctx->type == LOGFILE_TYPE_FILE || + file_ctx->type == LOGFILE_TYPE_UNIX_DGRAM || + file_ctx->type == LOGFILE_TYPE_UNIX_STREAM) + { + /* append \n for files only */ + MemBufferWriteString(buffer, "\n"); + SCMutexLock(&file_ctx->fp_mutex); + file_ctx->Write((const char *)MEMBUFFER_BUFFER(buffer), + MEMBUFFER_OFFSET(buffer), file_ctx); + SCMutexUnlock(&file_ctx->fp_mutex); + } +#ifdef HAVE_LIBHIREDIS + else if (file_ctx->type == LOGFILE_TYPE_REDIS) { + SCMutexLock(&file_ctx->fp_mutex); + LogFileWriteRedis(file_ctx, (const char *)MEMBUFFER_BUFFER(buffer), + MEMBUFFER_OFFSET(buffer)); + SCMutexUnlock(&file_ctx->fp_mutex); + } +#endif + + return 0; +} diff --git a/framework/src/suricata/src/util-logopenfile.h b/framework/src/suricata/src/util-logopenfile.h index d345475d..f0a123ac 100644 --- a/framework/src/suricata/src/util-logopenfile.h +++ b/framework/src/suricata/src/util-logopenfile.h @@ -26,6 +26,11 @@ #include "conf.h" /* ConfNode */ #include "tm-modules.h" /* LogFileCtx */ +#include "util-buffer.h" + +#ifdef HAVE_LIBHIREDIS +#include "hiredis/hiredis.h" +#endif typedef struct { uint16_t fileno; @@ -34,13 +39,43 @@ typedef struct { enum LogFileType { LOGFILE_TYPE_FILE, LOGFILE_TYPE_SYSLOG, LOGFILE_TYPE_UNIX_DGRAM, - LOGFILE_TYPE_UNIX_STREAM }; + LOGFILE_TYPE_UNIX_STREAM, + LOGFILE_TYPE_REDIS }; + +typedef struct SyslogSetup_ { + int alert_syslog_level; +} SyslogSetup; + +#ifdef HAVE_LIBHIREDIS +enum RedisMode { REDIS_LIST, REDIS_CHANNEL }; + +typedef struct RedisSetup_ { + enum RedisMode mode; + const char *command; + char *key; + int batch_size; + int batch_count; + char *server; + int port; + time_t tried; +} RedisSetup; +#endif /** Global structure for Output Context */ typedef struct LogFileCtx_ { union { FILE *fp; PcieFile *pcie_fp; +#ifdef HAVE_LIBHIREDIS + redisContext *redis; +#endif + }; + + union { + SyslogSetup syslog_setup; +#ifdef HAVE_LIBHIREDIS + RedisSetup redis_setup; +#endif }; int (*Write)(const char *buffer, int buffer_len, struct LogFileCtx_ *fp); @@ -56,6 +91,9 @@ typedef struct LogFileCtx_ { /** The name of the file */ char *filename; + /** Suricata sensor name */ + char *sensor_name; + /** Handle auto-connecting / reconnecting sockets */ int is_sock; int sock_type; @@ -92,8 +130,10 @@ typedef struct LogFileCtx_ { LogFileCtx *LogFileNewCtx(void); int LogFileFreeCtx(LogFileCtx *); +int LogFileWrite(LogFileCtx *file_ctx, MemBuffer *buffer); int SCConfLogOpenGeneric(ConfNode *conf, LogFileCtx *, const char *, int); +int SCConfLogOpenRedis(ConfNode *conf, LogFileCtx *log_ctx); int SCConfLogReopen(LogFileCtx *); #endif /* __UTIL_LOGOPENFILE_H__ */ diff --git a/framework/src/suricata/src/util-profiling-rules.c b/framework/src/suricata/src/util-profiling-rules.c index 2f4ec5c7..945bc8b3 100644 --- a/framework/src/suricata/src/util-profiling-rules.c +++ b/framework/src/suricata/src/util-profiling-rules.c @@ -38,10 +38,6 @@ #ifdef PROFILING -#ifndef MIN -#define MIN(a, b) (((a) < (b)) ? (a) : (b)) -#endif - /** * Extra data for rule profiling. */ @@ -85,6 +81,7 @@ extern int profiling_output_to_file; int profiling_rules_enabled = 0; static char *profiling_file_name = ""; static const char *profiling_file_mode = "a"; +static int profiling_rule_json = 0; /** * Sort orders for dumping profiled rules. @@ -182,6 +179,13 @@ void SCProfilingRulesGlobalInit(void) profiling_output_to_file = 1; } + if (ConfNodeChildValueIsTrue(conf, "json")) { +#ifdef HAVE_LIBJANSSON + profiling_rule_json = 1; +#else + SCLogWarning(SC_ERR_NO_JSON_SUPPORT, "no json support compiled in, using plain output"); +#endif + } } } } @@ -284,6 +288,134 @@ SCProfileSummarySortByMaxTicks(const void *a, const void *b) return s0->max > s1->max ? -1 : 1; } +#ifdef HAVE_LIBJANSSON + +static void DumpJson(FILE *fp, SCProfileSummary *summary, uint32_t count, uint64_t total_ticks) +{ + char timebuf[64]; + uint32_t i; + struct timeval tval; + + json_t *js = json_object(); + if (js == NULL) + return; + json_t *jsa = json_array(); + if (jsa == NULL) { + json_decref(js); + return; + } + + gettimeofday(&tval, NULL); + CreateIsoTimeString(&tval, timebuf, sizeof(timebuf)); + json_object_set_new(js, "timestamp", json_string(timebuf)); + + for (i = 0; i < count; i++) { + /* Stop dumping when we hit our first rule with 0 checks. Due + * to sorting this will be the beginning of all the rules with + * 0 checks. */ + if (summary[i].checks == 0) + break; + + json_t *jsm = json_object(); + if (jsm) { + json_object_set_new(jsm, "signature_id", json_integer(summary[i].sid)); + json_object_set_new(jsm, "gid", json_integer(summary[i].gid)); + json_object_set_new(jsm, "rev", json_integer(summary[i].rev)); + + json_object_set_new(jsm, "checks", json_integer(summary[i].checks)); + json_object_set_new(jsm, "matches", json_integer(summary[i].matches)); + + json_object_set_new(jsm, "ticks_total", json_integer(summary[i].ticks)); + json_object_set_new(jsm, "ticks_max", json_integer(summary[i].max)); + json_object_set_new(jsm, "ticks_avg", json_integer(summary[i].avgticks)); + json_object_set_new(jsm, "ticks_avg_match", json_integer(summary[i].avgticks_match)); + json_object_set_new(jsm, "ticks_avg_nomatch", json_integer(summary[i].avgticks_no_match)); + + double percent = (long double)summary[i].ticks / + (long double)total_ticks * 100; + json_object_set_new(jsm, "percent", json_integer(percent)); + json_array_append(jsa, jsm); + } + } + json_object_set_new(js, "rules", jsa); + + char *js_s = json_dumps(js, + JSON_PRESERVE_ORDER|JSON_COMPACT|JSON_ENSURE_ASCII| +#ifdef JSON_ESCAPE_SLASH + JSON_ESCAPE_SLASH +#else + 0 +#endif + ); + + if (unlikely(js_s == NULL)) + return; + fprintf(fp, "%s", js_s); + free(js_s); + json_decref(js); +} + +#endif /* HAVE_LIBJANSSON */ + +static void DumpText(FILE *fp, SCProfileSummary *summary, uint32_t count, uint64_t total_ticks) +{ + uint32_t i; + struct timeval tval; + struct tm *tms; + gettimeofday(&tval, NULL); + struct tm local_tm; + tms = SCLocalTime(tval.tv_sec, &local_tm); + + fprintf(fp, " ----------------------------------------------" + "----------------------------\n"); + fprintf(fp, " Date: %" PRId32 "/%" PRId32 "/%04d -- " + "%02d:%02d:%02d\n", tms->tm_mon + 1, tms->tm_mday, tms->tm_year + 1900, + tms->tm_hour,tms->tm_min, tms->tm_sec); + fprintf(fp, " ----------------------------------------------" + "----------------------------\n"); + fprintf(fp, " %-8s %-12s %-8s %-8s %-12s %-6s %-8s %-8s %-11s %-11s %-11s %-11s\n", "Num", "Rule", "Gid", "Rev", "Ticks", "%", "Checks", "Matches", "Max Ticks", "Avg Ticks", "Avg Match", "Avg No Match"); + fprintf(fp, " -------- " + "------------ " + "-------- " + "-------- " + "------------ " + "------ " + "-------- " + "-------- " + "----------- " + "----------- " + "----------- " + "-------------- " + "\n"); + for (i = 0; i < MIN(count, profiling_rules_limit); i++) { + + /* Stop dumping when we hit our first rule with 0 checks. Due + * to sorting this will be the beginning of all the rules with + * 0 checks. */ + if (summary[i].checks == 0) + break; + + double percent = (long double)summary[i].ticks / + (long double)total_ticks * 100; + fprintf(fp, + " %-8"PRIu32" %-12u %-8"PRIu32" %-8"PRIu32" %-12"PRIu64" %-6.2f %-8"PRIu64" %-8"PRIu64" %-11"PRIu64" %-11.2f %-11.2f %-11.2f\n", + i + 1, + summary[i].sid, + summary[i].gid, + summary[i].rev, + summary[i].ticks, + percent, + summary[i].checks, + summary[i].matches, + summary[i].max, + summary[i].avgticks, + summary[i].avgticks_match, + summary[i].avgticks_no_match); + } + + fprintf(fp,"\n"); +} + /** * \brief Dump rule profiling information to file * @@ -298,8 +430,6 @@ SCProfilingRuleDump(SCProfileDetectCtx *rules_ctx) if (rules_ctx == NULL) return; - struct timeval tval; - struct tm *tms; if (profiling_output_to_file == 1) { fp = fopen(profiling_file_name, profiling_file_mode); @@ -383,59 +513,15 @@ SCProfilingRuleDump(SCProfileDetectCtx *rules_ctx) SCProfileSummarySortByAvgTicksNoMatch); break; } - - gettimeofday(&tval, NULL); - struct tm local_tm; - tms = SCLocalTime(tval.tv_sec, &local_tm); - - fprintf(fp, " ----------------------------------------------" - "----------------------------\n"); - fprintf(fp, " Date: %" PRId32 "/%" PRId32 "/%04d -- " - "%02d:%02d:%02d\n", tms->tm_mon + 1, tms->tm_mday, tms->tm_year + 1900, - tms->tm_hour,tms->tm_min, tms->tm_sec); - fprintf(fp, " ----------------------------------------------" - "----------------------------\n"); - fprintf(fp, " %-8s %-12s %-8s %-8s %-12s %-6s %-8s %-8s %-11s %-11s %-11s %-11s\n", "Num", "Rule", "Gid", "Rev", "Ticks", "%", "Checks", "Matches", "Max Ticks", "Avg Ticks", "Avg Match", "Avg No Match"); - fprintf(fp, " -------- " - "------------ " - "-------- " - "-------- " - "------------ " - "------ " - "-------- " - "-------- " - "----------- " - "----------- " - "----------- " - "-------------- " - "\n"); - for (i = 0; i < MIN(count, profiling_rules_limit); i++) { - - /* Stop dumping when we hit our first rule with 0 checks. Due - * to sorting this will be the beginning of all the rules with - * 0 checks. */ - if (summary[i].checks == 0) - break; - - double percent = (long double)summary[i].ticks / - (long double)total_ticks * 100; - fprintf(fp, - " %-8"PRIu32" %-12u %-8"PRIu32" %-8"PRIu32" %-12"PRIu64" %-6.2f %-8"PRIu64" %-8"PRIu64" %-11"PRIu64" %-11.2f %-11.2f %-11.2f\n", - i + 1, - summary[i].sid, - summary[i].gid, - summary[i].rev, - summary[i].ticks, - percent, - summary[i].checks, - summary[i].matches, - summary[i].max, - summary[i].avgticks, - summary[i].avgticks_match, - summary[i].avgticks_no_match); +#ifdef HAVE_LIBJANSSON + if (profiling_rule_json) { + DumpJson(fp, summary, count, total_ticks); + } else +#endif + { + DumpText(fp, summary, count, total_ticks); } - fprintf(fp,"\n"); if (fp != stdout) fclose(fp); SCFree(summary); diff --git a/framework/src/suricata/src/util-runmodes.c b/framework/src/suricata/src/util-runmodes.c index c9b61e6a..a327a512 100644 --- a/framework/src/suricata/src/util-runmodes.c +++ b/framework/src/suricata/src/util-runmodes.c @@ -420,7 +420,7 @@ int RunModeSetLiveCaptureWorkers(ConfigIfaceParserFunc ConfigParser, for (ldev = 0; ldev < nlive; ldev++) { char *live_dev_c = NULL; - if (live_dev != NULL) { + if ((nlive <= 1) && (live_dev != NULL)) { aconf = ConfigParser(live_dev); live_dev_c = SCStrdup(live_dev); if (unlikely(live_dev_c == NULL)) { diff --git a/framework/src/suricata/src/util-validate.h b/framework/src/suricata/src/util-validate.h index b5c3da26..b7647a7d 100644 --- a/framework/src/suricata/src/util-validate.h +++ b/framework/src/suricata/src/util-validate.h @@ -96,11 +96,14 @@ } \ } while(0) +#define DEBUG_VALIDATE_BUG_ON(exp) BUG_ON((exp)) + #else /* DEBUG_VALIDATE */ #define DEBUG_ASSERT_FLOW_LOCKED(f) #define DEBUG_VALIDATE_FLOW(f) #define DEBUG_VALIDATE_PACKET(p) +#define DEBUG_VALIDATE_BUG_ON(exp) #endif /* DEBUG_VALIDATE */ |