diff options
Diffstat (limited to 'framework/src/suricata/src/output-json-email-common.c')
-rw-r--r-- | framework/src/suricata/src/output-json-email-common.c | 481 |
1 files changed, 337 insertions, 144 deletions
diff --git a/framework/src/suricata/src/output-json-email-common.c b/framework/src/suricata/src/output-json-email-common.c index 1efa9ce8..88cd3acf 100644 --- a/framework/src/suricata/src/output-json-email-common.c +++ b/framework/src/suricata/src/output-json-email-common.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2014 Open Information Security Foundation +/* Copyright (C) 2007-2015 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -19,6 +19,7 @@ * \file * * \author Tom DeCanio <td@npulsetech.com> + * \author Eric Leblond <eric@regit.org> * * Implements json common email logging portion of the engine. */ @@ -55,137 +56,290 @@ #ifdef HAVE_LIBJANSSON #include <jansson.h> +#define LOG_EMAIL_DEFAULT 0 +#define LOG_EMAIL_EXTENDED (1<<0) +#define LOG_EMAIL_ARRAY (1<<1) /* require array handling */ +#define LOG_EMAIL_COMMA (1<<2) /* require array handling */ +#define LOG_EMAIL_BODY_MD5 (1<<3) +#define LOG_EMAIL_SUBJECT_MD5 (1<<4) + +struct { + char *config_field; + char *email_field; + uint32_t flags; +} email_fields[] = { + { "reply_to", "reply-to", LOG_EMAIL_DEFAULT }, + { "bcc", "bcc", LOG_EMAIL_COMMA|LOG_EMAIL_EXTENDED }, + { "message_id", "message-id", LOG_EMAIL_EXTENDED }, + { "subject", "subject", LOG_EMAIL_EXTENDED }, + { "x_mailer", "x-mailer", LOG_EMAIL_EXTENDED }, + { "user_agent", "user-agent", LOG_EMAIL_EXTENDED }, + { "received", "received", LOG_EMAIL_ARRAY }, + { "x_originating_ip", "x-originating-ip", LOG_EMAIL_DEFAULT }, + { "in_reply_to", "in-reply-to", LOG_EMAIL_DEFAULT }, + { "references", "references", LOG_EMAIL_DEFAULT }, + { "importance", "importance", LOG_EMAIL_DEFAULT }, + { "priority", "priority", LOG_EMAIL_DEFAULT }, + { "sensitivity", "sensitivity", LOG_EMAIL_DEFAULT }, + { "organization", "organization", LOG_EMAIL_DEFAULT }, + { "content_md5", "content-md5", LOG_EMAIL_DEFAULT }, + { "date", "date", LOG_EMAIL_DEFAULT }, + { NULL, NULL, LOG_EMAIL_DEFAULT}, +}; + +static inline char *SkipWhiteSpaceTill(char *p, char *savep) +{ + char *sp = p; + if (unlikely(p == NULL)) { + return NULL; + } + while (((*sp == '\t') || (*sp == ' ')) && (sp < savep)) { + sp++; + } + return sp; +} + +static json_t* JsonEmailJsonArrayFromCommaList(const uint8_t *val, size_t len) +{ + json_t *ajs = json_array(); + if (likely(ajs != NULL)) { + char *savep = NULL; + char *p; + char *sp; + char *to_line = BytesToString((uint8_t *)val, len); + if (likely(to_line != NULL)) { + p = strtok_r(to_line, ",", &savep); + if (p == NULL) { + json_decref(ajs); + SCFree(to_line); + return NULL; + } + sp = SkipWhiteSpaceTill(p, savep); + json_array_append_new(ajs, json_string(sp)); + while ((p = strtok_r(NULL, ",", &savep)) != NULL) { + sp = SkipWhiteSpaceTill(p, savep); + json_array_append_new(ajs, json_string(sp)); + } + } + SCFree(to_line); + } + + return ajs; +} + + +#ifdef HAVE_NSS +static void JsonEmailLogJSONMd5(OutputJsonEmailCtx *email_ctx, json_t *js, SMTPTransaction *tx) +{ + if (email_ctx->flags & LOG_EMAIL_SUBJECT_MD5) { + MimeDecField *field; + MimeDecEntity *entity = tx->msg_tail; + if (entity == NULL) { + return; + } + field = MimeDecFindField(entity, "subject"); + if (field != NULL) { + unsigned char md5[MD5_LENGTH]; + char smd5[256]; + char *value = BytesToString((uint8_t *)field->value , field->value_len); + if (value) { + size_t i,x; + HASH_HashBuf(HASH_AlgMD5, md5, (unsigned char *)value, strlen(value)); + for (i = 0, x = 0; x < sizeof(md5); x++) { + i += snprintf(smd5 + i, 255 - i, "%02x", md5[x]); + } + json_object_set_new(js, "subject_md5", json_string(smd5)); + SCFree(value); + } + } + } + + if (email_ctx->flags & LOG_EMAIL_BODY_MD5) { + MimeDecParseState *mime_state = tx->mime_state; + if (mime_state && mime_state->md5_ctx && (mime_state->state_flag == PARSE_DONE)) { + size_t x; + int i; + char s[256]; + if (likely(s != NULL)) { + for (i = 0, x = 0; x < sizeof(mime_state->md5); x++) { + i += snprintf(s + i, 255-i, "%02x", mime_state->md5[x]); + } + json_object_set_new(js, "body_md5", json_string(s)); + } + } + } +} +#endif + +static int JsonEmailAddToJsonArray(const uint8_t *val, size_t len, void *data) +{ + json_t *ajs = data; + + if (ajs == NULL) + return 0; + char *value = BytesToString((uint8_t *)val, len); + json_array_append_new(ajs, json_string(value)); + SCFree(value); + return 1; +} + +static void JsonEmailLogJSONCustom(OutputJsonEmailCtx *email_ctx, json_t *js, SMTPTransaction *tx) +{ + int f = 0; + MimeDecField *field; + MimeDecEntity *entity = tx->msg_tail; + if (entity == NULL) { + return; + } + + while(email_fields[f].config_field) { + if (((email_ctx->fields & (1ULL<<f)) != 0) + || + ((email_ctx->flags & LOG_EMAIL_EXTENDED) && (email_fields[f].flags & LOG_EMAIL_EXTENDED)) + ) { + if (email_fields[f].flags & LOG_EMAIL_ARRAY) { + json_t *ajs = json_array(); + if (ajs) { + int found = MimeDecFindFieldsForEach(entity, email_fields[f].email_field, JsonEmailAddToJsonArray, ajs); + if (found > 0) { + json_object_set_new(js, email_fields[f].config_field, ajs); + } else { + json_decref(ajs); + } + } + } else if (email_fields[f].flags & LOG_EMAIL_COMMA) { + field = MimeDecFindField(entity, email_fields[f].email_field); + if (field) { + json_t *ajs = JsonEmailJsonArrayFromCommaList(field->value, field->value_len); + if (ajs) { + json_object_set_new(js, email_fields[f].config_field, ajs); + } + } + } else { + field = MimeDecFindField(entity, email_fields[f].email_field); + if (field != NULL) { + char *s = BytesToString((uint8_t *)field->value, + (size_t)field->value_len); + if (likely(s != NULL)) { + json_object_set_new(js, email_fields[f].config_field, json_string(s)); + SCFree(s); + } + } + } + + } + f++; + } +} + /* JSON format logging */ -static TmEcode JsonEmailLogJson(JsonEmailLogThread *aft, json_t *js, const Packet *p, Flow *f, void *state, void *vtx, uint64_t tx_id) +json_t *JsonEmailLogJsonData(const Flow *f, void *state, void *vtx, uint64_t tx_id) { SMTPState *smtp_state; MimeDecParseState *mime_state; MimeDecEntity *entity; - char *protos = NULL; json_t *sjs = json_object(); if (sjs == NULL) { - SCReturnInt(TM_ECODE_FAILED); + SCReturnPtr(NULL, "json_t"); } /* check if we have SMTP state or not */ - AppProto proto = FlowGetAppProtocol(p->flow); + AppProto proto = FlowGetAppProtocol(f); switch (proto) { case ALPROTO_SMTP: smtp_state = (SMTPState *)state; if (smtp_state == NULL) { SCLogDebug("no smtp state, so no request logging"); - SCReturnInt(TM_ECODE_FAILED); + SCReturnPtr(NULL, "json_t"); } SMTPTransaction *tx = vtx; mime_state = tx->mime_state; entity = tx->msg_tail; - protos = "smtp"; SCLogDebug("lets go mime_state %p, entity %p, state_flag %u", mime_state, entity, mime_state ? mime_state->state_flag : 0); break; default: /* don't know how we got here */ - SCReturnInt(TM_ECODE_FAILED); + SCReturnPtr(NULL, "json_t"); } if ((mime_state != NULL)) { if (entity == NULL) { - SCReturnInt(TM_ECODE_FAILED); + SCReturnPtr(NULL, "json_t"); } - if ((entity->header_flags & HDR_IS_LOGGED) == 0) { - MimeDecField *field; - //printf("email LOG\n"); - - /* From: */ - field = MimeDecFindField(entity, "from"); - if (field != NULL) { - char *s = BytesToString((uint8_t *)field->value, - (size_t)field->value_len); - if (likely(s != NULL)) { - //printf("From: \"%s\"\n", s); - json_object_set_new(sjs, "from", json_string(s)); - SCFree(s); - } + json_object_set_new(sjs, "status", + json_string(MimeDecParseStateGetStatus(mime_state))); + + MimeDecField *field; + + /* From: */ + field = MimeDecFindField(entity, "from"); + if (field != NULL) { + char *s = BytesToString((uint8_t *)field->value, + (size_t)field->value_len); + if (likely(s != NULL)) { + //printf("From: \"%s\"\n", s); + char * sp = SkipWhiteSpaceTill(s, s + strlen(s)); + json_object_set_new(sjs, "from", json_string(sp)); + SCFree(s); } + } - /* To: */ - char *to_line = NULL; - field = MimeDecFindField(entity, "to"); - if (field != NULL) { - json_t *js_to = json_array(); - if (likely(js_to != NULL)) { - to_line = BytesToString((uint8_t *)field->value, - (size_t)field->value_len); - if (likely(to_line != NULL)) { - char *savep = NULL; - char *p; - //printf("to_line:: TO: \"%s\" (%d)\n", to_line, strlen(to_line)); - p = strtok_r(to_line, ",", &savep); - //printf("got another addr: \"%s\"\n", p); - json_array_append_new(js_to, json_string(p)); - while ((p = strtok_r(NULL, ",", &savep)) != NULL) { - //printf("got another addr: \"%s\"\n", p); - json_array_append_new(js_to, json_string(&p[strspn(p, " ")])); - } - SCFree(to_line); - } - json_object_set_new(sjs, "to", js_to); - } + /* To: */ + field = MimeDecFindField(entity, "to"); + if (field != NULL) { + json_t *ajs = JsonEmailJsonArrayFromCommaList(field->value, field->value_len); + if (ajs) { + json_object_set_new(sjs, "to", ajs); } + } - /* Cc: */ - char *cc_line = NULL; - field = MimeDecFindField(entity, "cc"); - if (field != NULL) { - json_t *js_cc = json_array(); - if (likely(js_cc != NULL)) { - cc_line = BytesToString((uint8_t *)field->value, - (size_t)field->value_len); - if (likely(cc_line != NULL)) { - char *savep = NULL; - char *p; - //printf("cc_line:: CC: \"%s\" (%d)\n", to_line, strlen(to_line)); - p = strtok_r(cc_line, ",", &savep); - //printf("got another addr: \"%s\"\n", p); - json_array_append_new(js_cc, json_string(p)); - while ((p = strtok_r(NULL, ",", &savep)) != NULL) { - //printf("got another addr: \"%s\"\n", p); - json_array_append_new(js_cc, json_string(&p[strspn(p, " ")])); - } - SCFree(cc_line); - } - json_object_set_new(sjs, "cc", js_cc); - } + /* Cc: */ + field = MimeDecFindField(entity, "cc"); + if (field != NULL) { + json_t *ajs = JsonEmailJsonArrayFromCommaList(field->value, field->value_len); + if (ajs) { + json_object_set_new(sjs, "cc", ajs); } + } - /* Subject: */ - field = MimeDecFindField(entity, "subject"); - if (field != NULL) { - char *s = BytesToString((uint8_t *)field->value, (size_t) field->value_len); - if (likely(s != NULL)) { - //printf("Subject: \"%s\"\n", s); - json_object_set_new(sjs, "subject", json_string(s)); + if (mime_state->stack == NULL || mime_state->stack->top == NULL || mime_state->stack->top->data == NULL) + SCReturnPtr(NULL, "json_t"); + + entity = (MimeDecEntity *)mime_state->stack->top->data; + int attch_cnt = 0; + int url_cnt = 0; + json_t *js_attch = json_array(); + json_t *js_url = json_array(); + if (entity->url_list != NULL) { + MimeDecUrl *url; + for (url = entity->url_list; url != NULL; url = url->next) { + char *s = BytesToString((uint8_t *)url->url, + (size_t)url->url_len); + if (s != NULL) { + json_array_append_new(js_url, + json_string(s)); SCFree(s); + url_cnt += 1; } } + } + for (entity = entity->child; entity != NULL; entity = entity->next) { + if (entity->ctnt_flags & CTNT_IS_ATTACHMENT) { - entity->header_flags |= HDR_IS_LOGGED; - - if (mime_state->stack == NULL || mime_state->stack->top == NULL || mime_state->stack->top->data == NULL) - SCReturnInt(TM_ECODE_OK); - - entity = (MimeDecEntity *)mime_state->stack->top->data; - int attch_cnt = 0; - int url_cnt = 0; - json_t *js_attch = json_array(); - json_t *js_url = json_array(); + char *s = BytesToString((uint8_t *)entity->filename, + (size_t)entity->filename_len); + json_array_append_new(js_attch, + json_string(s)); + SCFree(s); + attch_cnt += 1; + } if (entity->url_list != NULL) { MimeDecUrl *url; for (url = entity->url_list; url != NULL; url = url->next) { char *s = BytesToString((uint8_t *)url->url, (size_t)url->url_len); if (s != NULL) { - //printf("URL: \"%s\"\n", s); json_array_append_new(js_url, json_string(s)); SCFree(s); @@ -193,74 +347,113 @@ static TmEcode JsonEmailLogJson(JsonEmailLogThread *aft, json_t *js, const Packe } } } - for (entity = entity->child; entity != NULL; entity = entity->next) { - if (entity->ctnt_flags & CTNT_IS_ATTACHMENT) { - - char *s = BytesToString((uint8_t *)entity->filename, - (size_t)entity->filename_len); - //printf("found attachment \"%s\"\n", s); - json_array_append_new(js_attch, - json_string(s)); - SCFree(s); - attch_cnt += 1; - } - if (entity->url_list != NULL) { - MimeDecUrl *url; - for (url = entity->url_list; url != NULL; url = url->next) { - char *s = BytesToString((uint8_t *)url->url, - (size_t)url->url_len); - if (s != NULL) { - //printf("URL: \"%s\"\n", s); - json_array_append_new(js_url, - json_string(s)); - SCFree(s); - url_cnt += 1; - } - } - } - } - if (attch_cnt > 0) { - json_object_set_new(sjs, "attachment", js_attch); - } else { - json_decref(js_attch); - } - if (url_cnt > 0) { - json_object_set_new(sjs, "url", js_url); - } else { - json_decref(js_url); - } - json_object_set_new(js, protos, sjs); - -// FLOWLOCK_UNLOCK(p->flow); - SCReturnInt(TM_ECODE_OK); } + if (attch_cnt > 0) { + json_object_set_new(sjs, "attachment", js_attch); + } else { + json_decref(js_attch); + } + if (url_cnt > 0) { + json_object_set_new(sjs, "url", js_url); + } else { + json_decref(js_url); + } + SCReturnPtr(sjs, "json_t"); } -// FLOWLOCK_UNLOCK(p->flow); - SCReturnInt(TM_ECODE_DONE); + json_decref(sjs); + SCReturnPtr(NULL, "json_t"); } -int JsonEmailLogger(ThreadVars *tv, void *thread_data, const Packet *p, Flow *f, void *state, void *tx, uint64_t tx_id) { - SCEnter(); - JsonEmailLogThread *jhl = (JsonEmailLogThread *)thread_data; - MemBuffer *buffer = (MemBuffer *)jhl->buffer; +/* JSON format logging */ +TmEcode JsonEmailLogJson(JsonEmailLogThread *aft, json_t *js, const Packet *p, Flow *f, void *state, void *vtx, uint64_t tx_id) +{ + json_t *sjs = JsonEmailLogJsonData(f, state, vtx, tx_id); + OutputJsonEmailCtx *email_ctx = aft->emaillog_ctx; + SMTPTransaction *tx = (SMTPTransaction *) vtx; + + if ((email_ctx->flags & LOG_EMAIL_EXTENDED) || (email_ctx->fields != 0)) + JsonEmailLogJSONCustom(email_ctx, sjs, tx); - json_t *js = CreateJSONHeader((Packet *)p, 1, "smtp"); - if (unlikely(js == NULL)) - return TM_ECODE_OK; +#ifdef HAVE_NSS + JsonEmailLogJSONMd5(email_ctx, sjs, tx); +#endif - /* reset */ - MemBufferReset(buffer); + if (sjs) { + json_object_set_new(js, "email", sjs); + SCReturnInt(TM_ECODE_OK); + } else + SCReturnInt(TM_ECODE_FAILED); +} + +json_t *JsonEmailAddMetadata(const Flow *f, uint32_t tx_id) +{ + SMTPState *smtp_state = (SMTPState *)FlowGetAppState(f); + if (smtp_state) { + SMTPTransaction *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_SMTP, smtp_state, tx_id); - if (JsonEmailLogJson(jhl, js, p, f, state, tx, tx_id) == TM_ECODE_OK) { - OutputJSONBuffer(js, jhl->emaillog_ctx->file_ctx, buffer); + if (tx) { + return JsonEmailLogJsonData(f, smtp_state, tx, tx_id); + } } - json_object_del(js, "smtp"); - json_object_clear(js); - json_decref(js); + return NULL; +} + + +void OutputEmailInitConf(ConfNode *conf, OutputJsonEmailCtx *email_ctx) +{ + if (conf) { + const char *extended = ConfNodeLookupChildValue(conf, "extended"); + + if (extended != NULL) { + if (ConfValIsTrue(extended)) { + email_ctx->flags = LOG_EMAIL_EXTENDED; + } + } + + email_ctx->fields = 0; + ConfNode *custom; + if ((custom = ConfNodeLookupChild(conf, "custom")) != NULL) { + ConfNode *field; + TAILQ_FOREACH(field, &custom->head, next) { + if (field != NULL) { + int f = 0; + while(email_fields[f].config_field) { + if ((strcmp(email_fields[f].config_field, + field->val) == 0) || + (strcasecmp(email_fields[f].email_field, + field->val) == 0)) + { + email_ctx->fields |= (1ULL<<f); + break; + } + f++; + } + } + } + } - SCReturnInt(TM_ECODE_OK); + email_ctx->flags = 0; + ConfNode *md5_conf; + if ((md5_conf = ConfNodeLookupChild(conf, "md5")) != NULL) { + ConfNode *field; + TAILQ_FOREACH(field, &md5_conf->head, next) { + if (field != NULL) { + if (strcmp("body", field->val) == 0) { + SCLogInfo("Going to log the md5 sum of email body"); + email_ctx->flags |= LOG_EMAIL_BODY_MD5; + } + if (strcmp("subject", field->val) == 0) { + SCLogInfo("Going to log the md5 sum of email subject"); + email_ctx->flags |= LOG_EMAIL_SUBJECT_MD5; + } + } + } + } + } + return; } + #endif |