diff options
author | Ashlee Young <ashlee@onosfw.com> | 2015-09-09 22:21:41 -0700 |
---|---|---|
committer | Ashlee Young <ashlee@onosfw.com> | 2015-09-09 22:21:41 -0700 |
commit | 8879b125d26e8db1a5633de5a9c692eb2d1c4f83 (patch) | |
tree | c7259d85a991b83dfa85ab2e339360669fc1f58e /framework/src/suricata/src/log-filestore.c | |
parent | 13d05bc8458758ee39cb829098241e89616717ee (diff) |
suricata checkin based on commit id a4bce14770beee46a537eda3c3f6e8e8565d5d0a
Change-Id: I9a214fa0ee95e58fc640e50bd604dac7f42db48f
Diffstat (limited to 'framework/src/suricata/src/log-filestore.c')
-rw-r--r-- | framework/src/suricata/src/log-filestore.c | 499 |
1 files changed, 499 insertions, 0 deletions
diff --git a/framework/src/suricata/src/log-filestore.c b/framework/src/suricata/src/log-filestore.c new file mode 100644 index 00000000..4244b9a4 --- /dev/null +++ b/framework/src/suricata/src/log-filestore.c @@ -0,0 +1,499 @@ +/* Copyright (C) 2007-2013 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Victor Julien <victor@inliniac.net> + * + */ + +#include "suricata-common.h" +#include "debug.h" +#include "detect.h" +#include "pkt-var.h" +#include "conf.h" + +#include "threadvars.h" +#include "tm-modules.h" + +#include "threads.h" + +#include "app-layer-parser.h" + +#include "detect-filemagic.h" + +#include "stream.h" + +#include "util-print.h" +#include "util-unittest.h" +#include "util-privs.h" +#include "util-debug.h" +#include "util-atomic.h" +#include "util-file.h" +#include "util-time.h" + +#include "output.h" + +#include "log-file.h" +#include "util-logopenfile.h" + +#include "app-layer-htp.h" +#include "app-layer-smtp.h" +#include "util-decode-mime.h" +#include "util-memcmp.h" +#include "stream-tcp-reassemble.h" + +#define MODULE_NAME "LogFilestoreLog" + +static char g_logfile_base_dir[PATH_MAX] = "/tmp"; + +typedef struct LogFilestoreLogThread_ { + LogFileCtx *file_ctx; + /** LogFilestoreCtx has the pointer to the file and a mutex to allow multithreading */ + uint32_t file_cnt; +} LogFilestoreLogThread; + +static void LogFilestoreMetaGetUri(FILE *fp, const Packet *p, const File *ff) +{ + HtpState *htp_state = (HtpState *)p->flow->alstate; + if (htp_state != NULL) { + htp_tx_t *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP, htp_state, ff->txid); + if (tx != NULL) { + HtpTxUserData *tx_ud = htp_tx_get_user_data(tx); + if (tx_ud->request_uri_normalized != NULL) { + PrintRawUriFp(fp, bstr_ptr(tx_ud->request_uri_normalized), + bstr_len(tx_ud->request_uri_normalized)); + } + return; + } + } + + fprintf(fp, "<unknown>"); +} + +static void LogFilestoreMetaGetHost(FILE *fp, const Packet *p, const File *ff) +{ + HtpState *htp_state = (HtpState *)p->flow->alstate; + if (htp_state != NULL) { + htp_tx_t *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP, htp_state, ff->txid); + if (tx != NULL && tx->request_hostname != NULL) { + PrintRawUriFp(fp, (uint8_t *)bstr_ptr(tx->request_hostname), + bstr_len(tx->request_hostname)); + return; + } + } + + fprintf(fp, "<unknown>"); +} + +static void LogFilestoreMetaGetReferer(FILE *fp, const Packet *p, const File *ff) +{ + HtpState *htp_state = (HtpState *)p->flow->alstate; + if (htp_state != NULL) { + htp_tx_t *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP, htp_state, ff->txid); + if (tx != NULL) { + htp_header_t *h = NULL; + h = (htp_header_t *)htp_table_get_c(tx->request_headers, + "Referer"); + if (h != NULL) { + PrintRawUriFp(fp, (uint8_t *)bstr_ptr(h->value), + bstr_len(h->value)); + return; + } + } + } + + fprintf(fp, "<unknown>"); +} + +static void LogFilestoreMetaGetUserAgent(FILE *fp, const Packet *p, const File *ff) +{ + HtpState *htp_state = (HtpState *)p->flow->alstate; + if (htp_state != NULL) { + htp_tx_t *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP, htp_state, ff->txid); + if (tx != NULL) { + htp_header_t *h = NULL; + h = (htp_header_t *)htp_table_get_c(tx->request_headers, + "User-Agent"); + if (h != NULL) { + PrintRawUriFp(fp, (uint8_t *)bstr_ptr(h->value), + bstr_len(h->value)); + return; + } + } + } + + fprintf(fp, "<unknown>"); +} + +static void LogFilestoreMetaGetSmtp(FILE *fp, const Packet *p, const File *ff) +{ + SMTPState *state = (SMTPState *) p->flow->alstate; + if (state != NULL) { + SMTPTransaction *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_SMTP, state, ff->txid); + if (tx == NULL || tx->msg_tail == NULL) + return; + + /* Message Id */ + if (tx->msg_tail->msg_id != NULL) { + fprintf(fp, "MESSAGE-ID: "); + PrintRawUriFp(fp, (uint8_t *) tx->msg_tail->msg_id, tx->msg_tail->msg_id_len); + fprintf(fp, "\n"); + } + + /* Sender */ + MimeDecField *field = MimeDecFindField(tx->msg_tail, "from"); + if (field != NULL) { + fprintf(fp, "SENDER: "); + PrintRawUriFp(fp, (uint8_t *) field->value, field->value_len); + fprintf(fp, "\n"); + } + } +} + +static void LogFilestoreLogCreateMetaFile(const Packet *p, const File *ff, char *filename, int ipver) { + char metafilename[PATH_MAX] = ""; + snprintf(metafilename, sizeof(metafilename), "%s.meta", filename); + FILE *fp = fopen(metafilename, "w+"); + if (fp != NULL) { + char timebuf[64]; + + CreateTimeString(&p->ts, timebuf, sizeof(timebuf)); + + fprintf(fp, "TIME: %s\n", timebuf); + if (p->pcap_cnt > 0) { + fprintf(fp, "PCAP PKT NUM: %"PRIu64"\n", p->pcap_cnt); + } + + char srcip[46], dstip[46]; + Port sp, dp; + switch (ipver) { + case AF_INET: + PrintInet(AF_INET, (const void *)GET_IPV4_SRC_ADDR_PTR(p), srcip, sizeof(srcip)); + PrintInet(AF_INET, (const void *)GET_IPV4_DST_ADDR_PTR(p), dstip, sizeof(dstip)); + break; + case AF_INET6: + PrintInet(AF_INET6, (const void *)GET_IPV6_SRC_ADDR(p), srcip, sizeof(srcip)); + PrintInet(AF_INET6, (const void *)GET_IPV6_DST_ADDR(p), dstip, sizeof(dstip)); + break; + default: + strlcpy(srcip, "<unknown>", sizeof(srcip)); + strlcpy(dstip, "<unknown>", sizeof(dstip)); + break; + } + sp = p->sp; + dp = p->dp; + + fprintf(fp, "SRC IP: %s\n", srcip); + fprintf(fp, "DST IP: %s\n", dstip); + fprintf(fp, "PROTO: %" PRIu32 "\n", p->proto); + if (PKT_IS_TCP(p) || PKT_IS_UDP(p)) { + fprintf(fp, "SRC PORT: %" PRIu16 "\n", sp); + fprintf(fp, "DST PORT: %" PRIu16 "\n", dp); + } + + fprintf(fp, "APP PROTO: %s\n", + AppProtoToString(p->flow->alproto)); + + /* Only applicable to HTTP traffic */ + if (p->flow->alproto == ALPROTO_HTTP) { + fprintf(fp, "HTTP URI: "); + LogFilestoreMetaGetUri(fp, p, ff); + fprintf(fp, "\n"); + fprintf(fp, "HTTP HOST: "); + LogFilestoreMetaGetHost(fp, p, ff); + fprintf(fp, "\n"); + fprintf(fp, "HTTP REFERER: "); + LogFilestoreMetaGetReferer(fp, p, ff); + fprintf(fp, "\n"); + fprintf(fp, "HTTP USER AGENT: "); + LogFilestoreMetaGetUserAgent(fp, p, ff); + fprintf(fp, "\n"); + } else if (p->flow->alproto == ALPROTO_SMTP) { + /* Only applicable to SMTP */ + LogFilestoreMetaGetSmtp(fp, p, ff); + } + + fprintf(fp, "FILENAME: "); + PrintRawUriFp(fp, ff->name, ff->name_len); + fprintf(fp, "\n"); + + fclose(fp); + } +} + +static void LogFilestoreLogCloseMetaFile(const File *ff) +{ + char filename[PATH_MAX] = ""; + snprintf(filename, sizeof(filename), "%s/file.%u", + g_logfile_base_dir, ff->file_id); + char metafilename[PATH_MAX] = ""; + snprintf(metafilename, sizeof(metafilename), "%s.meta", filename); + FILE *fp = fopen(metafilename, "a"); + if (fp != NULL) { + fprintf(fp, "MAGIC: %s\n", + ff->magic ? ff->magic : "<unknown>"); + + switch (ff->state) { + case FILE_STATE_CLOSED: + fprintf(fp, "STATE: CLOSED\n"); +#ifdef HAVE_NSS + if (ff->flags & FILE_MD5) { + fprintf(fp, "MD5: "); + size_t x; + for (x = 0; x < sizeof(ff->md5); x++) { + fprintf(fp, "%02x", ff->md5[x]); + } + fprintf(fp, "\n"); + } +#endif + break; + case FILE_STATE_TRUNCATED: + fprintf(fp, "STATE: TRUNCATED\n"); + break; + case FILE_STATE_ERROR: + fprintf(fp, "STATE: ERROR\n"); + break; + default: + fprintf(fp, "STATE: UNKNOWN\n"); + break; + } + fprintf(fp, "SIZE: %"PRIu64"\n", ff->size); + + fclose(fp); + } else { + SCLogInfo("opening %s failed: %s", metafilename, strerror(errno)); + } +} + +static int LogFilestoreLogger(ThreadVars *tv, void *thread_data, const Packet *p, const File *ff, const FileData *ffd, uint8_t flags) +{ + SCEnter(); + LogFilestoreLogThread *aft = (LogFilestoreLogThread *)thread_data; + char filename[PATH_MAX] = ""; + int file_fd = -1; + int ipver = -1; + + /* no flow, no htp state */ + if (p->flow == NULL) { + SCReturnInt(TM_ECODE_OK); + } + + if (PKT_IS_IPV4(p)) { + ipver = AF_INET; + } else if (PKT_IS_IPV6(p)) { + ipver = AF_INET6; + } else { + return 0; + } + + SCLogDebug("ff %p, ffd %p", ff, ffd); + + snprintf(filename, sizeof(filename), "%s/file.%u", + g_logfile_base_dir, ff->file_id); + + if (flags & OUTPUT_FILEDATA_FLAG_OPEN) { + aft->file_cnt++; + + /* create a .meta file that contains time, src/dst/sp/dp/proto */ + LogFilestoreLogCreateMetaFile(p, ff, filename, ipver); + + file_fd = open(filename, O_CREAT | O_TRUNC | O_NOFOLLOW | O_WRONLY, 0644); + if (file_fd == -1) { + SCLogDebug("failed to create file"); + return -1; + } + /* we can get called with a NULL ffd when we need to close */ + } else if (ffd != NULL) { + file_fd = open(filename, O_APPEND | O_NOFOLLOW | O_WRONLY); + if (file_fd == -1) { + SCLogDebug("failed to open file %s: %s", filename, strerror(errno)); + return -1; + } + } + + if (file_fd != -1) { + ssize_t r = write(file_fd, (const void *)ffd->data, (size_t)ffd->len); + if (r == -1) { + SCLogDebug("write failed: %s", strerror(errno)); + } + close(file_fd); + } + + if (flags & OUTPUT_FILEDATA_FLAG_CLOSE) { + LogFilestoreLogCloseMetaFile(ff); + } + + return 0; +} + +static TmEcode LogFilestoreLogThreadInit(ThreadVars *t, void *initdata, void **data) +{ + LogFilestoreLogThread *aft = SCMalloc(sizeof(LogFilestoreLogThread)); + if (unlikely(aft == NULL)) + return TM_ECODE_FAILED; + memset(aft, 0, sizeof(LogFilestoreLogThread)); + + if (initdata == NULL) + { + SCLogDebug("Error getting context for LogFilestore. \"initdata\" argument NULL"); + SCFree(aft); + return TM_ECODE_FAILED; + } + + /* Use the Ouptut Context (file pointer and mutex) */ + aft->file_ctx = ((OutputCtx *)initdata)->data; + + struct stat stat_buf; + if (stat(g_logfile_base_dir, &stat_buf) != 0) { + int ret; + ret = mkdir(g_logfile_base_dir, S_IRWXU|S_IXGRP|S_IRGRP); + if (ret != 0) { + int err = errno; + if (err != EEXIST) { + SCLogError(SC_ERR_LOGDIR_CONFIG, + "Cannot create file drop directory %s: %s", + g_logfile_base_dir, strerror(err)); + exit(EXIT_FAILURE); + } + } else { + SCLogInfo("Created file drop directory %s", + g_logfile_base_dir); + } + + } + + *data = (void *)aft; + return TM_ECODE_OK; +} + +static TmEcode LogFilestoreLogThreadDeinit(ThreadVars *t, void *data) +{ + LogFilestoreLogThread *aft = (LogFilestoreLogThread *)data; + if (aft == NULL) { + return TM_ECODE_OK; + } + + /* clear memory */ + memset(aft, 0, sizeof(LogFilestoreLogThread)); + + SCFree(aft); + return TM_ECODE_OK; +} + +static void LogFilestoreLogExitPrintStats(ThreadVars *tv, void *data) +{ + LogFilestoreLogThread *aft = (LogFilestoreLogThread *)data; + if (aft == NULL) { + return; + } + + SCLogInfo("(%s) Files extracted %" PRIu32 "", tv->name, aft->file_cnt); +} + +/** + * \internal + * + * \brief deinit the log ctx and write out the waldo + * + * \param output_ctx output context to deinit + */ +static void LogFilestoreLogDeInitCtx(OutputCtx *output_ctx) +{ + LogFileCtx *logfile_ctx = (LogFileCtx *)output_ctx->data; + LogFileFreeCtx(logfile_ctx); + SCFree(output_ctx); + +} + +/** \brief Create a new http log LogFilestoreCtx. + * \param conf Pointer to ConfNode containing this loggers configuration. + * \return NULL if failure, LogFilestoreCtx* to the file_ctx if succesful + * */ +static OutputCtx *LogFilestoreLogInitCtx(ConfNode *conf) +{ + LogFileCtx *logfile_ctx = LogFileNewCtx(); + if (logfile_ctx == NULL) { + SCLogDebug("Could not create new LogFilestoreCtx"); + return NULL; + } + + OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx)); + if (unlikely(output_ctx == NULL)) + return NULL; + + output_ctx->data = NULL; + output_ctx->DeInit = LogFilestoreLogDeInitCtx; + + char *s_default_log_dir = NULL; + s_default_log_dir = ConfigGetLogDirectory(); + + const char *s_base_dir = NULL; + s_base_dir = ConfNodeLookupChildValue(conf, "log-dir"); + if (s_base_dir == NULL || strlen(s_base_dir) == 0) { + strlcpy(g_logfile_base_dir, + s_default_log_dir, sizeof(g_logfile_base_dir)); + } else { + if (PathIsAbsolute(s_base_dir)) { + strlcpy(g_logfile_base_dir, + s_base_dir, sizeof(g_logfile_base_dir)); + } else { + snprintf(g_logfile_base_dir, sizeof(g_logfile_base_dir), + "%s/%s", s_default_log_dir, s_base_dir); + } + } + + const char *force_magic = ConfNodeLookupChildValue(conf, "force-magic"); + if (force_magic != NULL && ConfValIsTrue(force_magic)) { + FileForceMagicEnable(); + SCLogInfo("forcing magic lookup for stored files"); + } + + const char *force_md5 = ConfNodeLookupChildValue(conf, "force-md5"); + if (force_md5 != NULL && ConfValIsTrue(force_md5)) { +#ifdef HAVE_NSS + FileForceMd5Enable(); + SCLogInfo("forcing md5 calculation for stored files"); +#else + SCLogInfo("md5 calculation requires linking against libnss"); +#endif + } + SCLogInfo("storing files in %s", g_logfile_base_dir); + + SCReturnPtr(output_ctx, "OutputCtx"); +} + +void TmModuleLogFilestoreRegister (void) +{ + tmm_modules[TMM_FILESTORE].name = MODULE_NAME; + tmm_modules[TMM_FILESTORE].ThreadInit = LogFilestoreLogThreadInit; + tmm_modules[TMM_FILESTORE].Func = NULL; + tmm_modules[TMM_FILESTORE].ThreadExitPrintStats = LogFilestoreLogExitPrintStats; + tmm_modules[TMM_FILESTORE].ThreadDeinit = LogFilestoreLogThreadDeinit; + tmm_modules[TMM_FILESTORE].RegisterTests = NULL; + tmm_modules[TMM_FILESTORE].cap_flags = 0; + tmm_modules[TMM_FILESTORE].flags = TM_FLAG_LOGAPI_TM; + tmm_modules[TMM_FILESTORE].priority = 10; + + OutputRegisterFiledataModule(MODULE_NAME, "file", LogFilestoreLogInitCtx, + LogFilestoreLogger); + OutputRegisterFiledataModule(MODULE_NAME, "file-store", LogFilestoreLogInitCtx, + LogFilestoreLogger); + + SCLogDebug("registered"); +} |