diff options
26 files changed, 1715 insertions, 1 deletions
diff --git a/framework/src/suricata/qa/docker/buildbot.cfg b/framework/src/suricata/qa/docker/buildbot.cfg index bce14d22..b2063ac9 100644 --- a/framework/src/suricata/qa/docker/buildbot.cfg +++ b/framework/src/suricata/qa/docker/buildbot.cfg @@ -126,7 +126,7 @@ pcaps_list = [ os.path.join(PCAP_PATH, pcap) for pcap in pcaps_list if pcap.ends factory_stress_pcap = SuriBuildFactory() # run the tests (note that this will require that 'trial' is installed) factory_stress_pcap.addStep(ShellCommand(command=["./autogen.sh"])) -factory_stress_pcap.addStep(ShellCommand(command=["./configure","--enable-debug-validation"])) +factory_stress_pcap.addStep(ShellCommand(command=["./configure","--enable-debug-validation"],env={"CFLAGS" : "-fsanitize=address -fno-omit-frame-pointer"})) factory_stress_pcap.addStep(ShellCommand(command=["make"])) factory_stress_pcap.addStep(ShellCommand(command=["sudo", "make","install"])) factory_stress_pcap.addStep(ShellCommand(command=["sudo", "rm", "-f", "/usr/local/etc/suricata/suricata.yaml"])) diff --git a/framework/src/suricata/scripts/setup-app-layer-detect.sh b/framework/src/suricata/scripts/setup-app-layer-detect.sh new file mode 100755 index 00000000..ef3b741e --- /dev/null +++ b/framework/src/suricata/scripts/setup-app-layer-detect.sh @@ -0,0 +1,233 @@ +#! /bin/sh +# +# Script to provision a new application layer detector and parser. + +set -e + +function usage() { + cat <<EOF + +usage: $0 <protocol name> + +This script will provision content inspection for app-layer decoded +buffers. + +Examples: + + $0 DNP3 + $0 Gopher + +EOF +} + +fail_if_exists() { + path="$1" + if test -e "${path}"; then + echo "error: ${path} already exists." + exit 1 + fi +} + +function copy_template_file() { + src="$1" + dst="$2" + + echo "Creating ${dst}." + + sed -e "s/TEMPLATE/${protoname_upper}/g" \ + -e "s/template/${protoname_lower}/g" \ + -e "s/Template/${protoname}/g" > ${dst} < ${src} +} + +function copy_templates() { + detect_h_dst="src/detect-${protoname_lower}-buffer.h" + detect_c_dst="src/detect-${protoname_lower}-buffer.c" + detect_engine_h_dst="src/detect-engine-${protoname_lower}.h" + detect_engine_c_dst="src/detect-engine-${protoname_lower}.c" + + fail_if_exists ${detect_h_dst} + fail_if_exists ${detect_c_dst} + fail_if_exists ${detect_engine_h_dst} + fail_if_exists ${detect_engine_c_dst} + + copy_template_file "src/detect-template-buffer.h" ${detect_h_dst} + copy_template_file "src/detect-template-buffer.c" ${detect_c_dst} + copy_template_file "src/detect-engine-template.h" ${detect_engine_h_dst} + copy_template_file "src/detect-engine-template.c" ${detect_engine_c_dst} +} + +function patch_makefile_am() { + filename="src/Makefile.am" + echo "Patching ${filename}." + ed -s ${filename} > /dev/null <<EOF +/^detect-engine-template.c +t- +s/template/${protoname_lower}/g +/^detect-template-buffer.c +t- +s/template/${protoname_lower}/g +w +EOF +} + +function patch_detect_engine_content_inspection_h() { + filename="src/detect-engine-content-inspection.h" + echo "Patching ${filename}." + ed -s ${filename} > /dev/null <<EOF +/DETECT_ENGINE_CONTENT_INSPECTION_MODE_TEMPLATE_BUFFER +t- +s/TEMPLATE/${protoname_upper}/ +w +EOF +} + +function patch_detect_engine_state_h() { + filename="src/detect-engine-state.h" + echo "Patching ${filename}." + ed -s ${filename} > /dev/null <<EOF +/#define DE_STATE_FLAG_TEMPLATE_BUFFER_INSPECT +t- +s/TEMPLATE/${protoname_upper}/ +w +EOF +} + +function patch_detect_engine_c() { + filename="src/detect-engine.c" + echo "Patching ${filename}." + ed -s ${filename} > /dev/null <<EOF +/#include "detect-engine-template.h" +t- +s/template/${protoname_lower}/ +w +/ALPROTO_TEMPLATE +-2 +.,+6t- +-6 +.,+6s/Template/${protoname}/g +-6 +.,+6s/TEMPLATE/${protoname_upper}/g ++6 +/ALPROTO_TEMPLATE +-2 +.,+6t- +-6 +.,+6s/Template/${protoname}/g +-6 +.,+6s/TEMPLATE/${protoname_upper}/g +w +EOF + + ed -s ${filename} > /dev/null <<EOF +/case DETECT_SM_LIST_TEMPLATE_BUFFER_MATCH +.,+1t- +- +s/TEMPLATE/${protoname_upper}/g ++ +s/template/${protoname_lower}/g +w +EOF +} + +function patch_detect_parse_c() { + filename="src/detect-parse.c" + echo "Patching ${filename}." + ed -s ${filename} > /dev/null <<EOF +/\/\* Template\. \*\/ +.,+4t- +-4s/Template/${protoname}/g ++1s/TEMPLATE/${protoname_upper}/g +w +EOF +} + +function patch_detect_c() { + filename="src/detect.c" + echo "Patching ${filename}." + ed -s ${filename} > /dev/null <<EOF +/#include "detect-template-buffer.h" +t- +s/template/${protoname_lower}/ +/case ALPROTO_TEMPLATE +.,+3t- +-3 +s/ALPROTO_TEMPLATE/ALPROTO_${protoname_upper}/g ++ +s/template/${protoname_lower}/g ++ +s/TEMPLATE/${protoname_upper}/g ++2 +/ALPROTO_TEMPLATE +.,+3t- +-3 +.,+s/TEMPLATE/${protoname_upper}/g ++ +s/template/${protoname_lower}/g ++3 +/SIG_MASK_REQUIRE_TEMPLATE_STATE +.t- +s/TEMPLATE/${protoname_upper}/g +/DetectTemplateBufferRegister +t- +s/Template/${protoname}/ +w +EOF +} + +function patch_detect_h() { + filename="src/detect.h" + echo "Patching ${filename}." + ed -s ${filename} > /dev/null <<EOF +/DETECT_SM_LIST_TEMPLATE_BUFFER_MATCH +t- +s/TEMPLATE/${protoname_upper}/ +/SIG_MASK_REQUIRE_TEMPLATE_STATE +t- +s/TEMPLATE/${protoname_upper}/ +/DETECT_AL_TEMPLATE_BUFFER +t- +s/TEMPLATE/${protoname_upper}/ +w +EOF +} + +protoname="$1" + +if [ "${protoname}" = "" ]; then + usage + exit 1 +fi + +protoname_lower=$(printf ${protoname} | tr '[:upper:]' '[:lower:]') +protoname_upper=$(printf ${protoname} | tr '[:lower:]' '[:upper:]') + +copy_templates +patch_makefile_am +patch_detect_engine_content_inspection_h +patch_detect_engine_state_h +patch_detect_engine_c +patch_detect_parse_c +patch_detect_c +patch_detect_h + +cat <<EOF + +The following files have been created and linked into the build: + + detect-${protoname_lower}-buffer.h detect-${protoname_lower}-buffer.c + + The setup for the content inspection modifier keyword. + + detect-engine-${protoname_lower}.h detect-engine-${protoname_lower}.c + + The content inspection engine. + +Please fix in src/detect-engine-state.h the values for: + DE_STATE_FLAG_${protoname_upper}_BUFFER_INSPECT + DE_STATE_FLAG_TEMPLATE_BUFFER_INSPECT + +Please fix in src/detect.h the values for: + SIG_MASK_REQUIRE_${protoname_upper}_STATE + SIG_MASK_REQUIRE_TEMPLATE_STATE + +EOF diff --git a/framework/src/suricata/scripts/setup-app-layer-logger.sh b/framework/src/suricata/scripts/setup-app-layer-logger.sh new file mode 100755 index 00000000..be32c393 --- /dev/null +++ b/framework/src/suricata/scripts/setup-app-layer-logger.sh @@ -0,0 +1,154 @@ +#! /bin/sh + +set -e + +function usage() { + cat <<EOF + +usage: $0 <protocol name> + +This script will provision a new JSON application layer transaction +logger for the protocol name specified on the command line. This is +done by copying and patching src/output-json-template.h and +src/output-json-template.c then link the new files into the build +system. + +It is required that the application layer parser has already been +provisioned by the setup-app-layer.sh script. + +Examples: + + $0 DNP3 + $0 Gopher + +EOF +} + +fail_if_exists() { + path="$1" + if test -e "${path}"; then + echo "error: ${path} already exists." + exit 1 + fi +} + +function copy_template_file() { + src="$1" + dst="$2" + + echo "Creating ${dst}." + + sed -e "s/TEMPLATE/${protoname_upper}/g" \ + -e "s/template/${protoname_lower}/g" \ + -e "s/Template/${protoname}/g" > ${dst} < ${src} +} + +function copy_templates() { + src_h="src/output-json-template.h" + dst_h="src/output-json-${protoname_lower}.h" + src_c="src/output-json-template.c" + dst_c="src/output-json-${protoname_lower}.c" + + fail_if_exists ${dst_h} + fail_if_exists ${dst_c} + + copy_template_file ${src_h} ${dst_h} + copy_template_file ${src_c} ${dst_c} +} + +function patch_makefile_am() { + filename="src/Makefile.am" + echo "Patching ${filename}." + ed -s ${filename} > /dev/null <<EOF +/output-json-template.c +t- +s/template/${protoname_lower}/ +w +EOF +} + +function patch_suricata_c() { + filename="src/suricata.c" + echo "Patching ${filename}." + ed -s ${filename} > /dev/null <<EOF +/#include "output-json-template.h" +t- +s/template/${protoname_lower}/ +/TmModuleJsonTemplateLogRegister +- +.,+t- +- +.,+s/Template/${protoname}/ +w +EOF +} + +patch_tm_modules_c() { + filename="src/tm-modules.c" + echo "Patching ${filename}." + ed -s ${filename} > /dev/null <<EOF +/TMM_JSONTEMPLATELOG +t- +s/TEMPLATE/${protoname_upper} +w +EOF +} + +patch_tm_threads_common_h() { + filename="src/tm-threads-common.h" + echo "Patching ${filename}." + ed -s ${filename} > /dev/null <<EOF +/TMM_JSONTEMPLATELOG +t- +s/TEMPLATE/${protoname_upper} +w +EOF +} + +patch_suricata_yaml_in() { + filename="suricata.yaml.in" + echo "Patching ${filename}." + ed -s ${filename} > /dev/null <<EOF +/eve-log: +/types: +a + - ${protoname_lower} +. +w +EOF +} + +protoname="$1" + +if [ "${protoname}" = "" ]; then + usage + exit 1 +fi + +protoname_lower=$(printf ${protoname} | tr '[:upper:]' '[:lower:]') +protoname_upper=$(printf ${protoname} | tr '[:lower:]' '[:upper:]') + +# Requires that the protocol has already been setup. +if ! grep -q "ALPROTO_${protoname_upper}" src/app-layer-protos.h; then + echo "error: no app-layer parser exists for ALPROTO_${protoname_upper}." + exit 1 +fi + +copy_templates +patch_makefile_am +patch_suricata_c +patch_tm_modules_c +patch_tm_threads_common_h +patch_suricata_yaml_in + +cat <<EOF + +A JSON application layer transaction logger for the protocol +${protoname} has now been set in the files: + + src/output-json-${protoname_lower}.h + src/output-json-${protoname_lower}.c + +and should now build cleanly. Try running 'make'. + +EOF diff --git a/framework/src/suricata/scripts/setup-app-layer.sh b/framework/src/suricata/scripts/setup-app-layer.sh new file mode 100755 index 00000000..b24b5e61 --- /dev/null +++ b/framework/src/suricata/scripts/setup-app-layer.sh @@ -0,0 +1,166 @@ +#! /bin/sh +# +# Script to provision a new application layer detector and parser. + +set -e +#set -x + +function usage() { + cat <<EOF + +usage: $0 <protocol name> + +This script will provision a new app-layer parser for the protocol +name specified on the command line. This is done by copying and +patching src/app-layer-template.[ch] then linking the new files into +the build system. + +Examples: + + $0 DNP3 + $0 Gopher + +EOF +} + +fail_if_exists() { + path="$1" + if test -e "${path}"; then + echo "error: ${path} already exists." + exit 1 + fi +} + +function copy_template_file() { + src="$1" + dst="$2" + + echo "Creating ${dst}." + + sed -e "s/TEMPLATE/${protoname_upper}/g" \ + -e "s/template/${protoname_lower}/g" \ + -e "s/Template/${protoname}/g" > ${dst} < ${src} +} + +function copy_app_layer_templates { + src_h="src/app-layer-template.h" + dst_h="src/app-layer-${protoname_lower}.h" + src_c="src/app-layer-template.c" + dst_c="src/app-layer-${protoname_lower}.c" + + fail_if_exists ${dst_h} + fail_if_exists ${dst_c} + + copy_template_file ${src_h} ${dst_h} + copy_template_file ${src_c} ${dst_c} +} + +function patch_makefile_am { + filename="src/Makefile.am" + echo "Patching ${filename}." + ed -s ${filename} > /dev/null <<EOF +/app-layer-template +t- +s/template/${protoname_lower}/g +w +EOF +} + +function patch_app_layer_protos_h { + filename="src/app-layer-protos.h" + echo "Patching ${filename}." + ed -s ${filename} > /dev/null <<EOF +/ALPROTO_TEMPLATE +t- +s/TEMPLATE/${protoname_upper}/ +w +EOF +} + +function patch_app_layer_protos_c { + filename="src/app-layer-protos.c" + echo "Patching ${filename}." + ed -s ${filename} > /dev/null <<EOF +/case ALPROTO_TEMPLATE +.,+2t- +-2 +s/TEMPLATE/${protoname_upper}/ ++ +s/template/${protoname_lower}/ +w +EOF +} + +function patch_app_layer_detect_proto_c() { + filename="src/app-layer-detect-proto.c" + echo "Patching ${filename}." + ed -s ${filename} > /dev/null <<EOF +/== ALPROTO_TEMPLATE +.,+t- +-,.s/TEMPLATE/${protoname_upper}/ ++3 +/== ALPROTO_TEMPLATE +.,+t- +-,.s/TEMPLATE/${protoname_upper}/ ++3 +w +EOF +} + +function patch_app_layer_parser_c() { + filename="src/app-layer-parser.c" + echo "Patching ${filename}." + ed -s ${filename} > /dev/null <<EOF +/#include "app-layer-template.h" +t- +s/template/${protoname_lower}/ +/RegisterTemplateParsers +t- +s/Template/${protoname}/ +w +EOF +} + +function patch_suricata_yaml_in() { + filename="suricata.yaml.in" + echo "Patching ${filename}." + ed -s ${filename} > /dev/null <<EOF +/^app-layer: +/protocols: +a + ${protoname_lower}: + enabled: yes +. +w +EOF +} + +protoname="$1" + +if [ "${protoname}" = "" ]; then + usage + exit 1 +fi + +protoname_lower=$(printf ${protoname} | tr '[:upper:]' '[:lower:]') +protoname_upper=$(printf ${protoname} | tr '[:lower:]' '[:upper:]') + +copy_app_layer_templates +patch_makefile_am +patch_app_layer_protos_h +patch_app_layer_protos_c +patch_app_layer_detect_proto_c +patch_app_layer_parser_c +patch_suricata_yaml_in + +cat <<EOF + +An application detector and parser for the protocol ${protoname} has +now been setup in the files: + + src/app-layer-${protoname_lower}.h + src/app-layer-${protoname_lower}.c + +and should now build cleanly. Try running 'make'. + +EOF diff --git a/framework/src/suricata/src/Makefile.am b/framework/src/suricata/src/Makefile.am index de99d312..674043f5 100644 --- a/framework/src/suricata/src/Makefile.am +++ b/framework/src/suricata/src/Makefile.am @@ -33,6 +33,7 @@ app-layer-protos.c app-layer-protos.h \ app-layer-smb2.c app-layer-smb2.h \ app-layer-smb.c app-layer-smb.h \ app-layer-smtp.c app-layer-smtp.h \ +app-layer-template.c app-layer-template.h \ app-layer-ssh.c app-layer-ssh.h \ app-layer-ssl.c app-layer-ssl.h \ app-layer-tls-handshake.c app-layer-tls-handshake.h \ @@ -122,6 +123,7 @@ detect-engine-siggroup.c detect-engine-siggroup.h \ detect-engine-sigorder.c detect-engine-sigorder.h \ detect-engine-state.c detect-engine-state.h \ detect-engine-tag.c detect-engine-tag.h \ +detect-engine-template.c detect-engine-template.h \ detect-engine-threshold.c detect-engine-threshold.h \ detect-engine-uri.c detect-engine-uri.h \ detect-fast-pattern.c detect-fast-pattern.h \ @@ -194,6 +196,7 @@ detect-ssl-version.c detect-ssl-version.h \ detect-stream_size.c detect-stream_size.h \ detect-tag.c detect-tag.h \ detect-template.c detect-template.h \ +detect-template-buffer.c detect-template-buffer.h \ detect-threshold.c detect-threshold.h \ detect-tls.c detect-tls.h \ detect-tls-version.c detect-tls-version.h \ @@ -250,6 +253,7 @@ output-json-smtp.c output-json-smtp.h \ output-json-ssh.c output-json-ssh.h \ output-json-stats.c output-json-stats.h \ output-json-tls.c output-json-tls.h \ +output-json-template.c output-json-template.h \ output-lua.c output-lua.h \ output-packet.c output-packet.h \ output-stats.c output-stats.h \ diff --git a/framework/src/suricata/src/app-layer-detect-proto.c b/framework/src/suricata/src/app-layer-detect-proto.c index ed029e5e..221b50ff 100644 --- a/framework/src/suricata/src/app-layer-detect-proto.c +++ b/framework/src/suricata/src/app-layer-detect-proto.c @@ -687,6 +687,8 @@ void AppLayerProtoDetectPrintProbingParsers(AppLayerProtoDetectProbingParser *pp printf(" alproto: ALPROTO_DNS\n"); else if (pp_pe->alproto == ALPROTO_MODBUS) printf(" alproto: ALPROTO_MODBUS\n"); + else if (pp_pe->alproto == ALPROTO_TEMPLATE) + printf(" alproto: ALPROTO_TEMPLATE\n"); else printf("impossible\n"); @@ -738,6 +740,8 @@ void AppLayerProtoDetectPrintProbingParsers(AppLayerProtoDetectProbingParser *pp printf(" alproto: ALPROTO_DNS\n"); else if (pp_pe->alproto == ALPROTO_MODBUS) printf(" alproto: ALPROTO_MODBUS\n"); + else if (pp_pe->alproto == ALPROTO_TEMPLATE) + printf(" alproto: ALPROTO_TEMPLATE\n"); else printf("impossible\n"); diff --git a/framework/src/suricata/src/app-layer-parser.c b/framework/src/suricata/src/app-layer-parser.c index 134f9909..2650863e 100644 --- a/framework/src/suricata/src/app-layer-parser.c +++ b/framework/src/suricata/src/app-layer-parser.c @@ -58,6 +58,7 @@ #include "app-layer-dns-udp.h" #include "app-layer-dns-tcp.h" #include "app-layer-modbus.h" +#include "app-layer-template.h" #include "conf.h" #include "util-spm.h" @@ -1106,6 +1107,7 @@ void AppLayerParserRegisterProtocolParsers(void) RegisterDNSUDPParsers(); RegisterDNSTCPParsers(); RegisterModbusParsers(); + RegisterTemplateParsers(); /** IMAP */ AppLayerProtoDetectRegisterProtocol(ALPROTO_IMAP, "imap"); diff --git a/framework/src/suricata/src/app-layer-protos.c b/framework/src/suricata/src/app-layer-protos.c index 0b8ed17b..e8875643 100644 --- a/framework/src/suricata/src/app-layer-protos.c +++ b/framework/src/suricata/src/app-layer-protos.c @@ -75,6 +75,9 @@ const char *AppProtoToString(AppProto alproto) case ALPROTO_MODBUS: proto_name = "modbus"; break; + case ALPROTO_TEMPLATE: + proto_name = "template"; + break; case ALPROTO_FAILED: #ifdef UNITTESTS case ALPROTO_TEST: diff --git a/framework/src/suricata/src/app-layer-protos.h b/framework/src/suricata/src/app-layer-protos.h index 79973661..aff90e9b 100644 --- a/framework/src/suricata/src/app-layer-protos.h +++ b/framework/src/suricata/src/app-layer-protos.h @@ -42,6 +42,7 @@ enum AppProtoEnum { ALPROTO_DNS, ALPROTO_MODBUS, + ALPROTO_TEMPLATE, /* used by the probing parser when alproto detection fails * permanently for that particular stream */ diff --git a/framework/src/suricata/src/app-layer-template.c b/framework/src/suricata/src/app-layer-template.c new file mode 100644 index 00000000..87b64b73 --- /dev/null +++ b/framework/src/suricata/src/app-layer-template.c @@ -0,0 +1,533 @@ +/* Copyright (C) 2015 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file Template application layer detector and parser for learning and + * template pruposes. + * + * This template implements a simple application layer for something + * like the echo protocol running on port 7. + */ + +#include "suricata-common.h" +#include "stream.h" + +#include "util-unittest.h" + +#include "app-layer-detect-proto.h" +#include "app-layer-parser.h" + +#include "app-layer-template.h" + +/* The default port to probe for echo traffic if not provided in the + * configuration file. */ +#define TEMPLATE_DEFAULT_PORT "7" + +/* The minimum size for an echo message. For some protocols this might + * be the size of a header. */ +#define TEMPLATE_MIN_FRAME_LEN 1 + +/* Enum of app-layer events for an echo protocol. Normally you might + * have events for errors in parsing data, like unexpected data being + * received. For echo we'll make something up, and log an app-layer + * level alert if an empty message is received. + * + * Example rule: + * + * alert template any any -> any any (msg:"SURCATA Template empty message"; \ + * app-layer-event:template.empty_message; sid:X; rev:Y;) + */ +enum { + TEMPLATE_DECODER_EVENT_EMPTY_MESSAGE, +}; + +SCEnumCharMap template_decoder_event_table[] = { + {"EMPTY_MESSAGE", TEMPLATE_DECODER_EVENT_EMPTY_MESSAGE}, +}; + +static TemplateTransaction *TemplateTxAlloc(TemplateState *echo) +{ + TemplateTransaction *tx = SCCalloc(1, sizeof(TemplateTransaction)); + if (unlikely(tx == NULL)) { + return NULL; + } + + /* Increment the transaction ID on the state each time one is + * allocated. */ + tx->tx_id = echo->transaction_max++; + + TAILQ_INSERT_TAIL(&echo->tx_list, tx, next); + + return tx; +} + +static void TemplateTxFree(void *tx) +{ + TemplateTransaction *templatetx = tx; + + if (templatetx->request_buffer != NULL) { + SCFree(templatetx->request_buffer); + } + + if (templatetx->response_buffer != NULL) { + SCFree(templatetx->response_buffer); + } + + AppLayerDecoderEventsFreeEvents(&templatetx->decoder_events); + + SCFree(tx); +} + +static void *TemplateStateAlloc(void) +{ + SCLogNotice("Allocating template state."); + TemplateState *state = SCCalloc(1, sizeof(TemplateState)); + if (unlikely(state == NULL)) { + return NULL; + } + TAILQ_INIT(&state->tx_list); + return state; +} + +static void TemplateStateFree(void *state) +{ + TemplateState *template_state = state; + TemplateTransaction *tx; + SCLogNotice("Freeing template state."); + while ((tx = TAILQ_FIRST(&template_state->tx_list)) != NULL) { + TAILQ_REMOVE(&template_state->tx_list, tx, next); + TemplateTxFree(tx); + } + SCFree(template_state); +} + +/** + * \brief Callback from the application layer to have a transaction freed. + * + * \param state a void pointer to the TemplateState object. + * \param tx_id the transaction ID to free. + */ +static void TemplateStateTxFree(void *state, uint64_t tx_id) +{ + TemplateState *echo = state; + TemplateTransaction *tx = NULL, *ttx; + + SCLogNotice("Freeing transaction %"PRIu64, tx_id); + + TAILQ_FOREACH_SAFE(tx, &echo->tx_list, next, ttx) { + + /* Continue if this is not the transaction we are looking + * for. */ + if (tx->tx_id != tx_id) { + continue; + } + + /* Remove and free the transaction. */ + TAILQ_REMOVE(&echo->tx_list, tx, next); + TemplateTxFree(tx); + return; + } + + SCLogNotice("Transaction %"PRIu64" not found.", tx_id); +} + +static int TemplateStateGetEventInfo(const char *event_name, int *event_id, + AppLayerEventType *event_type) +{ + *event_id = SCMapEnumNameToValue(event_name, template_decoder_event_table); + if (*event_id == -1) { + SCLogError(SC_ERR_INVALID_ENUM_MAP, "event \"%s\" not present in " + "template enum map table.", event_name); + /* This should be treated as fatal. */ + return -1; + } + + *event_type = APP_LAYER_EVENT_TYPE_TRANSACTION; + + return 0; +} + +static AppLayerDecoderEvents *TemplateGetEvents(void *state, uint64_t tx_id) +{ + TemplateState *template_state = state; + TemplateTransaction *tx; + + TAILQ_FOREACH(tx, &template_state->tx_list, next) { + if (tx->tx_id == tx_id) { + return tx->decoder_events; + } + } + + return NULL; +} + +static int TemplateHasEvents(void *state) +{ + TemplateState *echo = state; + return echo->events; +} + +/** + * \brief Probe the input to see if it looks like echo. + * + * \retval ALPROTO_TEMPLATE if it looks like echo, otherwise + * ALPROTO_UNKNOWN. + */ +static AppProto TemplateProbingParser(uint8_t *input, uint32_t input_len, + uint32_t *offset) +{ + /* Very simple test - if there is input, this is echo. */ + if (input_len >= TEMPLATE_MIN_FRAME_LEN) { + SCLogNotice("Detected as ALPROTO_TEMPLATE."); + return ALPROTO_TEMPLATE; + } + + SCLogNotice("Protocol not detected as ALPROTO_TEMPLATE."); + return ALPROTO_UNKNOWN; +} + +static int TemplateParseRequest(Flow *f, void *state, + AppLayerParserState *pstate, uint8_t *input, uint32_t input_len, + void *local_data) +{ + TemplateState *echo = state; + + SCLogNotice("Parsing echo request: len=%"PRIu32, input_len); + + /* Likely connection closed, we can just return here. */ + if ((input == NULL || input_len == 0) && + AppLayerParserStateIssetFlag(pstate, APP_LAYER_PARSER_EOF)) { + return 0; + } + + /* Probably don't want to create a transaction in this case + * either. */ + if (input == NULL || input_len == 0) { + return 0; + } + + /* Normally you would parse out data here and store it in the + * transaction object, but as this is echo, we'll just record the + * request data. */ + + /* Also, if this protocol may have a "protocol data unit" span + * multiple chunks of data, which is always a possibility with + * TCP, you may need to do some buffering here. + * + * For the sake of simplicity, buffering is left out here, but + * even for an echo protocol we may want to buffer until a new + * line is seen, assuming its text based. + */ + + /* Allocate a transaction. + * + * But note that if a "protocol data unit" is not received in one + * chunk of data, and the buffering is done on the transaction, we + * may need to look for the transaction that this newly recieved + * data belongs to. + */ + TemplateTransaction *tx = TemplateTxAlloc(echo); + if (unlikely(tx == NULL)) { + SCLogNotice("Failed to allocate new Template tx."); + goto end; + } + SCLogNotice("Allocated Template tx %"PRIu64".", tx->tx_id); + + /* Make a copy of the request. */ + tx->request_buffer = SCCalloc(1, input_len); + if (unlikely(tx->request_buffer == NULL)) { + goto end; + } + memcpy(tx->request_buffer, input, input_len); + tx->request_buffer_len = input_len; + + /* Here we check for an empty message and create an app-layer + * event. */ + if ((input_len == 1 && tx->request_buffer[0] == '\n') || + (input_len == 2 && tx->request_buffer[0] == '\r')) { + SCLogNotice("Creating event for empty message."); + AppLayerDecoderEventsSetEventRaw(&tx->decoder_events, + TEMPLATE_DECODER_EVENT_EMPTY_MESSAGE); + echo->events++; + } + +end: + return 0; +} + +static int TemplateParseResponse(Flow *f, void *state, AppLayerParserState *pstate, + uint8_t *input, uint32_t input_len, void *local_data) +{ + TemplateState *echo = state; + TemplateTransaction *tx = NULL, *ttx;; + + SCLogNotice("Parsing Template response."); + + /* Likely connection closed, we can just return here. */ + if ((input == NULL || input_len == 0) && + AppLayerParserStateIssetFlag(pstate, APP_LAYER_PARSER_EOF)) { + return 0; + } + + /* Probably don't want to create a transaction in this case + * either. */ + if (input == NULL || input_len == 0) { + return 0; + } + + /* Look up the existing transaction for this response. In the case + * of echo, it will be the most recent transaction on the + * TemplateState object. */ + + /* We should just grab the last transaction, but this is to + * illustrate how you might traverse the transaction list to find + * the transaction associated with this response. */ + TAILQ_FOREACH(ttx, &echo->tx_list, next) { + tx = ttx; + } + + if (tx == NULL) { + SCLogNotice("Failed to find transaction for response on echo state %p.", + echo); + goto end; + } + + SCLogNotice("Found transaction %"PRIu64" for response on echo state %p.", + tx->tx_id, echo); + + /* If the protocol requires multiple chunks of data to complete, you may + * run into the case where you have existing response data. + * + * In this case, we just log that there is existing data and free it. But + * you might want to realloc the buffer and append the data. + */ + if (tx->response_buffer != NULL) { + SCLogNotice("WARNING: Transaction already has response data, " + "existing data will be overwritten."); + SCFree(tx->response_buffer); + } + + /* Make a copy of the response. */ + tx->response_buffer = SCCalloc(1, input_len); + if (unlikely(tx->response_buffer == NULL)) { + goto end; + } + memcpy(tx->response_buffer, input, input_len); + tx->response_buffer_len = input_len; + + /* Set the response_done flag for transaction state checking in + * TemplateGetStateProgress(). */ + tx->response_done = 1; + +end: + return 0; +} + +static uint64_t TemplateGetTxCnt(void *state) +{ + TemplateState *echo = state; + SCLogNotice("Current tx count is %"PRIu64".", echo->transaction_max); + return echo->transaction_max; +} + +static void *TemplateGetTx(void *state, uint64_t tx_id) +{ + TemplateState *echo = state; + TemplateTransaction *tx; + + SCLogNotice("Requested tx ID %"PRIu64".", tx_id); + + TAILQ_FOREACH(tx, &echo->tx_list, next) { + if (tx->tx_id == tx_id) { + SCLogNotice("Transaction %"PRIu64" found, returning tx object %p.", + tx_id, tx); + return tx; + } + } + + SCLogNotice("Transaction ID %"PRIu64" not found.", tx_id); + return NULL; +} + +/** + * \brief Called by the application layer. + * + * In most cases 1 can be returned here. + */ +static int TemplateGetAlstateProgressCompletionStatus(uint8_t direction) { + return 1; +} + +/** + * \brief Return the state of a transaction in a given direction. + * + * In the case of the echo protocol, the existence of a transaction + * means that the request is done. However, some protocols that may + * need multiple chunks of data to complete the request may need more + * than just the existence of a transaction for the request to be + * considered complete. + * + * For the response to be considered done, the response for a request + * needs to be seen. The response_done flag is set on response for + * checking here. + */ +static int TemplateGetStateProgress(void *tx, uint8_t direction) +{ + TemplateTransaction *echotx = tx; + + SCLogNotice("Transaction progress requested for tx ID %"PRIu64 + ", direction=0x%02x", echotx->tx_id, direction); + + if (direction & STREAM_TOCLIENT && echotx->response_done) { + return 1; + } + else if (direction & STREAM_TOSERVER) { + /* For echo, just the existence of the transaction means the + * request is done. */ + return 1; + } + + return 0; +} + +/** + * \brief ??? + */ +static DetectEngineState *TemplateGetTxDetectState(void *vtx) +{ + TemplateTransaction *tx = vtx; + return tx->de_state; +} + +/** + * \brief ??? + */ +static int TemplateSetTxDetectState(void *state, void *vtx, + DetectEngineState *s) +{ + TemplateTransaction *tx = vtx; + tx->de_state = s; + return 0; +} + +void RegisterTemplateParsers(void) +{ + char *proto_name = "template"; + + /* Check if Template TCP detection is enabled. If it does not exist in + * the configuration file then it will be enabled by default. */ + if (AppLayerProtoDetectConfProtoDetectionEnabled("tcp", proto_name)) { + + SCLogNotice("Template TCP protocol detection enabled."); + + AppLayerProtoDetectRegisterProtocol(ALPROTO_TEMPLATE, proto_name); + + if (RunmodeIsUnittests()) { + + SCLogNotice("Unittest mode, registeringd default configuration."); + AppLayerProtoDetectPPRegister(IPPROTO_TCP, TEMPLATE_DEFAULT_PORT, + ALPROTO_TEMPLATE, 0, TEMPLATE_MIN_FRAME_LEN, STREAM_TOSERVER, + TemplateProbingParser); + + } + else { + + if (!AppLayerProtoDetectPPParseConfPorts("tcp", IPPROTO_TCP, + proto_name, ALPROTO_TEMPLATE, 0, TEMPLATE_MIN_FRAME_LEN, + TemplateProbingParser)) { + SCLogNotice("No echo app-layer configuration, enabling echo" + " detection TCP detection on port %s.", + TEMPLATE_DEFAULT_PORT); + AppLayerProtoDetectPPRegister(IPPROTO_TCP, + TEMPLATE_DEFAULT_PORT, ALPROTO_TEMPLATE, 0, + TEMPLATE_MIN_FRAME_LEN, STREAM_TOSERVER, + TemplateProbingParser); + } + + } + + } + + else { + SCLogNotice("Protocol detecter and parser disabled for Template."); + return; + } + + if (AppLayerParserConfParserEnabled("udp", proto_name)) { + + SCLogNotice("Registering Template protocol parser."); + + /* Register functions for state allocation and freeing. A + * state is allocated for every new Template flow. */ + AppLayerParserRegisterStateFuncs(IPPROTO_TCP, ALPROTO_TEMPLATE, + TemplateStateAlloc, TemplateStateFree); + + /* Register request parser for parsing frame from server to client. */ + AppLayerParserRegisterParser(IPPROTO_TCP, ALPROTO_TEMPLATE, + STREAM_TOSERVER, TemplateParseRequest); + + /* Register response parser for parsing frames from server to client. */ + AppLayerParserRegisterParser(IPPROTO_TCP, ALPROTO_TEMPLATE, STREAM_TOCLIENT, + TemplateParseResponse); + + /* Register a function to be called by the application layer + * when a transaction is to be freed. */ + AppLayerParserRegisterTxFreeFunc(IPPROTO_TCP, ALPROTO_TEMPLATE, + TemplateStateTxFree); + + /* Register a function to return the current transaction count. */ + AppLayerParserRegisterGetTxCnt(IPPROTO_TCP, ALPROTO_TEMPLATE, TemplateGetTxCnt); + + /* Transaction handling. */ + AppLayerParserRegisterGetStateProgressCompletionStatus(IPPROTO_TCP, + ALPROTO_TEMPLATE, TemplateGetAlstateProgressCompletionStatus); + AppLayerParserRegisterGetStateProgressFunc(IPPROTO_TCP, + ALPROTO_TEMPLATE, TemplateGetStateProgress); + AppLayerParserRegisterGetTx(IPPROTO_TCP, ALPROTO_TEMPLATE, + TemplateGetTx); + + /* Application layer event handling. */ + AppLayerParserRegisterHasEventsFunc(IPPROTO_TCP, ALPROTO_TEMPLATE, + TemplateHasEvents); + + /* What is this being registered for? */ + AppLayerParserRegisterDetectStateFuncs(IPPROTO_TCP, ALPROTO_TEMPLATE, + NULL, TemplateGetTxDetectState, TemplateSetTxDetectState); + + AppLayerParserRegisterGetEventInfo(IPPROTO_TCP, ALPROTO_TEMPLATE, + TemplateStateGetEventInfo); + AppLayerParserRegisterGetEventsFunc(IPPROTO_TCP, ALPROTO_TEMPLATE, + TemplateGetEvents); + } + else { + SCLogNotice("Template protocol parsing disabled."); + } + +#ifdef UNITTESTS + AppLayerParserRegisterProtocolUnittests(IPPROTO_TCP, ALPROTO_TEMPLATE, + TemplateParserRegisterTests); +#endif +} + +#ifdef UNITTESTS +#endif + +void TemplateParserRegisterTests(void) +{ +#ifdef UNITTESTS +#endif +} diff --git a/framework/src/suricata/src/app-layer-template.h b/framework/src/suricata/src/app-layer-template.h new file mode 100644 index 00000000..4e58fa89 --- /dev/null +++ b/framework/src/suricata/src/app-layer-template.h @@ -0,0 +1,69 @@ +/* Copyright (C) 2015 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +#ifndef __APP_LAYER_TEMPLATE_H__ +#define __APP_LAYER_TEMPLATE_H__ + +#include "detect-engine-state.h" + +#include "queue.h" + +void RegisterTemplateParsers(void); +void TemplateParserRegisterTests(void); + +typedef struct TemplateTransaction_ { + + uint64_t tx_id; /*<< Internal transaction ID. */ + + AppLayerDecoderEvents *decoder_events; /*<< Application layer + * events that occurred + * while parsing this + * transaction. */ + + uint8_t *request_buffer; + uint32_t request_buffer_len; + + uint8_t *response_buffer; + uint32_t response_buffer_len; + + uint8_t response_done; /*<< Flag to be set when the response is + * seen. */ + + DetectEngineState *de_state; + + TAILQ_ENTRY(TemplateTransaction_) next; + +} TemplateTransaction; + +typedef struct TemplateState_ { + + TAILQ_HEAD(, TemplateTransaction_) tx_list; /**< List of Template transactions + * associated with this + * state. */ + + uint64_t transaction_max; /**< A count of the number of + * transactions created. The + * transaction ID for each transaction + * is allocted by incrementing this + * value. */ + + uint16_t events; /**< Number of application layer events created + * for this state. */ + +} TemplateState; + +#endif /* __APP_LAYER_TEMPLATE_H__ */ diff --git a/framework/src/suricata/src/detect-engine-content-inspection.h b/framework/src/suricata/src/detect-engine-content-inspection.h index 6d3accaa..dc0b5026 100644 --- a/framework/src/suricata/src/detect-engine-content-inspection.h +++ b/framework/src/suricata/src/detect-engine-content-inspection.h @@ -50,6 +50,7 @@ enum { DETECT_ENGINE_CONTENT_INSPECTION_MODE_HRHHD, DETECT_ENGINE_CONTENT_INSPECTION_MODE_DNSQUERY, DETECT_ENGINE_CONTENT_INSPECTION_MODE_FD_SMTP, + DETECT_ENGINE_CONTENT_INSPECTION_MODE_TEMPLATE_BUFFER, }; int DetectEngineContentInspection(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, diff --git a/framework/src/suricata/src/detect-engine-state.h b/framework/src/suricata/src/detect-engine-state.h index 51f67448..c8944cff 100644 --- a/framework/src/suricata/src/detect-engine-state.h +++ b/framework/src/suricata/src/detect-engine-state.h @@ -81,6 +81,7 @@ #define DE_STATE_FLAG_FD_SMTP_INSPECT (1 << 21) #define DE_STATE_FLAG_DNSREQUEST_INSPECT (1 << 22) #define DE_STATE_FLAG_DNSRESPONSE_INSPECT (1 << 23) +#define DE_STATE_FLAG_TEMPLATE_BUFFER_INSPECT (1 << 24) /* state flags */ #define DETECT_ENGINE_STATE_FLAG_FILE_STORE_DISABLED 0x0001 diff --git a/framework/src/suricata/src/detect-engine-template.c b/framework/src/suricata/src/detect-engine-template.c new file mode 100644 index 00000000..49c29c6a --- /dev/null +++ b/framework/src/suricata/src/detect-engine-template.c @@ -0,0 +1,46 @@ +/* Copyright (C) 2015 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +#include "suricata-common.h" +#include "stream.h" +#include "detect-engine-content-inspection.h" + +#include "app-layer-template.h" + +int DetectEngineInspectTemplateBuffer(ThreadVars *tv, DetectEngineCtx *de_ctx, + DetectEngineThreadCtx *det_ctx, Signature *s, Flow *f, uint8_t flags, + void *alstate, void *txv, uint64_t tx_id) +{ + TemplateTransaction *tx = (TemplateTransaction *)txv; + int ret = 0; + + if (flags & STREAM_TOSERVER && tx->request_buffer != NULL) { + ret = DetectEngineContentInspection(de_ctx, det_ctx, s, + s->sm_lists[DETECT_SM_LIST_TEMPLATE_BUFFER_MATCH], f, + tx->request_buffer, tx->request_buffer_len, 0, + DETECT_ENGINE_CONTENT_INSPECTION_MODE_TEMPLATE_BUFFER, NULL); + } + else if (flags & STREAM_TOCLIENT && tx->response_buffer != NULL) { + ret = DetectEngineContentInspection(de_ctx, det_ctx, s, + s->sm_lists[DETECT_SM_LIST_TEMPLATE_BUFFER_MATCH], f, + tx->response_buffer, tx->response_buffer_len, 0, + DETECT_ENGINE_CONTENT_INSPECTION_MODE_TEMPLATE_BUFFER, NULL); + } + + SCLogNotice("Returning %d.", ret); + return ret; +} diff --git a/framework/src/suricata/src/detect-engine-template.h b/framework/src/suricata/src/detect-engine-template.h new file mode 100644 index 00000000..61eb8d01 --- /dev/null +++ b/framework/src/suricata/src/detect-engine-template.h @@ -0,0 +1,25 @@ +/* Copyright (C) 2015 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +#ifndef __DETECT_TEMPLATE_ENGINE_H__ +#define __DETECT_TEMPLATE_ENGINE_H__ + +int DetectEngineInspectTemplateBuffer(ThreadVars *, DetectEngineCtx *, + DetectEngineThreadCtx *, Signature *, Flow *, uint8_t, void *, void *, + uint64_t); + +#endif /* __DETECT_TEMPLATE_ENGINE_H__ */ diff --git a/framework/src/suricata/src/detect-engine.c b/framework/src/suricata/src/detect-engine.c index 7de04969..c6e1a83f 100644 --- a/framework/src/suricata/src/detect-engine.c +++ b/framework/src/suricata/src/detect-engine.c @@ -62,6 +62,7 @@ #include "detect-engine-dns.h" #include "detect-engine-modbus.h" #include "detect-engine-filedata-smtp.h" +#include "detect-engine-template.h" #include "detect-engine.h" #include "detect-engine-state.h" @@ -285,6 +286,13 @@ void DetectEngineRegisterAppInspectionEngines(void) DE_STATE_FLAG_FD_SMTP_INSPECT, 0, DetectEngineInspectSMTPFiledata }, + /* Template. */ + { IPPROTO_TCP, + ALPROTO_TEMPLATE, + DETECT_SM_LIST_TEMPLATE_BUFFER_MATCH, + DE_STATE_FLAG_TEMPLATE_BUFFER_INSPECT, + 0, + DetectEngineInspectTemplateBuffer }, }; struct tmp_t data_toclient[] = { @@ -352,6 +360,13 @@ void DetectEngineRegisterAppInspectionEngines(void) DE_STATE_FLAG_DNSRESPONSE_INSPECT, 1, DetectEngineInspectDnsResponse }, + /* Template. */ + { IPPROTO_TCP, + ALPROTO_TEMPLATE, + DETECT_SM_LIST_TEMPLATE_BUFFER_MATCH, + DE_STATE_FLAG_TEMPLATE_BUFFER_INSPECT, + 1, + DetectEngineInspectTemplateBuffer }, }; size_t i; @@ -2630,6 +2645,9 @@ const char *DetectSigmatchListEnumToString(enum DetectSigmatchListEnum type) case DETECT_SM_LIST_MODBUS_MATCH: return "modbus"; + case DETECT_SM_LIST_TEMPLATE_BUFFER_MATCH: + return "template_buffer"; + case DETECT_SM_LIST_POSTMATCH: return "post-match"; diff --git a/framework/src/suricata/src/detect-parse.c b/framework/src/suricata/src/detect-parse.c index d077791e..fedfebe8 100644 --- a/framework/src/suricata/src/detect-parse.c +++ b/framework/src/suricata/src/detect-parse.c @@ -1479,6 +1479,11 @@ static Signature *SigInitHelper(DetectEngineCtx *de_ctx, char *sigstr, if (sig->sm_lists[DETECT_SM_LIST_HRHHDMATCH]) sig->flags |= SIG_FLAG_STATE_MATCH; + /* Template. */ + if (sig->sm_lists[DETECT_SM_LIST_TEMPLATE_BUFFER_MATCH]) { + sig->flags |= SIG_FLAG_STATE_MATCH; + } + /* DNS */ if (sig->sm_lists[DETECT_SM_LIST_DNSQUERYNAME_MATCH]) sig->flags |= SIG_FLAG_STATE_MATCH; diff --git a/framework/src/suricata/src/detect-template-buffer.c b/framework/src/suricata/src/detect-template-buffer.c new file mode 100644 index 00000000..64f948c2 --- /dev/null +++ b/framework/src/suricata/src/detect-template-buffer.c @@ -0,0 +1,165 @@ +/* Copyright (C) 2015 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file Set up of the "template_buffer" keyword to allow content inspections + * on the decoded template application layer buffers. + */ + +#include "suricata-common.h" +#include "detect.h" +#include "app-layer-template.h" + +static int DetectTemplateBufferSetup(DetectEngineCtx *, Signature *, char *); +static void DetectTemplateBufferRegisterTests(void); + +void DetectTemplateBufferRegister(void) +{ + sigmatch_table[DETECT_AL_TEMPLATE_BUFFER].name = "template_buffer"; + sigmatch_table[DETECT_AL_TEMPLATE_BUFFER].desc = + "Template content modififier to match on the template buffers"; + sigmatch_table[DETECT_AL_TEMPLATE_BUFFER].alproto = ALPROTO_TEMPLATE; + sigmatch_table[DETECT_AL_TEMPLATE_BUFFER].Setup = DetectTemplateBufferSetup; + sigmatch_table[DETECT_AL_TEMPLATE_BUFFER].RegisterTests = + DetectTemplateBufferRegisterTests; + + sigmatch_table[DETECT_AL_TEMPLATE_BUFFER].flags |= SIGMATCH_NOOPT; + sigmatch_table[DETECT_AL_TEMPLATE_BUFFER].flags |= SIGMATCH_PAYLOAD; + + SCLogNotice("Template application layer detect registered."); +} + +static int DetectTemplateBufferSetup(DetectEngineCtx *de_ctx, Signature *s, + char *str) +{ + s->list = DETECT_SM_LIST_TEMPLATE_BUFFER_MATCH; + s->alproto = ALPROTO_TEMPLATE; + return 0; +} + +#ifdef UNITTESTS + +#include "util-unittest.h" +#include "util-unittest-helper.h" +#include "app-layer-parser.h" +#include "detect-engine.h" +#include "detect-parse.h" +#include "flow-util.h" +#include "stream-tcp.h" + +static int DetectTemplateBufferTest(void) +{ + AppLayerParserThreadCtx *alp_tctx = AppLayerParserThreadCtxAlloc(); + DetectEngineThreadCtx *det_ctx = NULL; + DetectEngineCtx *de_ctx = NULL; + Flow f; + Packet *p; + TcpSession tcp; + ThreadVars tv; + Signature *s; + + int result = 0; + + uint8_t request[] = "Hello World!"; + + /* Setup flow. */ + memset(&f, 0, sizeof(Flow)); + memset(&tcp, 0, sizeof(TcpSession)); + memset(&tv, 0, sizeof(ThreadVars)); + p = UTHBuildPacket(request, sizeof(request), IPPROTO_TCP); + FLOW_INITIALIZE(&f); + f.alproto = ALPROTO_TEMPLATE; + f.protoctx = (void *)&tcp; + f.proto = IPPROTO_TCP; + f.flags |= FLOW_IPV4; + p->flow = &f; + p->flags |= PKT_HAS_FLOW | PKT_STREAM_EST; + p->flowflags |= FLOW_PKT_TOSERVER | FLOW_PKT_ESTABLISHED; + StreamTcpInitConfig(TRUE); + + de_ctx = DetectEngineCtxInit(); + if (de_ctx == NULL) { + goto end; + } + + /* This rule should match. */ + s = DetectEngineAppendSig(de_ctx, + "alert tcp any any -> any any (" + "msg:\"TEMPLATE Test Rule\"; " + "template_buffer; content:\"World!\"; " + "sid:1; rev:1;)"); + if (s == NULL) { + goto end; + } + + /* This rule should not match. */ + s = DetectEngineAppendSig(de_ctx, + "alert tcp any any -> any any (" + "msg:\"TEMPLATE Test Rule\"; " + "template_buffer; content:\"W0rld!\"; " + "sid:2; rev:1;)"); + if (s == NULL) { + goto end; + } + + SigGroupBuild(de_ctx); + DetectEngineThreadCtxInit(&tv, (void *)de_ctx, (void *)&det_ctx); + + SCMutexLock(&f.m); + AppLayerParserParse(alp_tctx, &f, ALPROTO_TEMPLATE, STREAM_TOSERVER, + request, sizeof(request)); + SCMutexUnlock(&f.m); + + /* Check that we have app-layer state. */ + if (f.alstate == NULL) { + goto end; + } + + SigMatchSignatures(&tv, de_ctx, det_ctx, p); + if (!PacketAlertCheck(p, 1)) { + goto end; + } + if (PacketAlertCheck(p, 2)) { + goto end; + } + + result = 1; +end: + /* Cleanup. */ + if (alp_tctx != NULL) + AppLayerParserThreadCtxFree(alp_tctx); + if (det_ctx != NULL) + DetectEngineThreadCtxDeinit(&tv, det_ctx); + if (de_ctx != NULL) + SigGroupCleanup(de_ctx); + if (de_ctx != NULL) + DetectEngineCtxFree(de_ctx); + StreamTcpFreeConfig(TRUE); + FLOW_DESTROY(&f); + UTHFreePacket(p); + + return result; +} + +#endif + +static void DetectTemplateBufferRegisterTests(void) +{ +#ifdef UNITTESTS + UtRegisterTest("DetectTemplateBufferTest", DetectTemplateBufferTest, 1); +#endif /* UNITTESTS */ +} diff --git a/framework/src/suricata/src/detect-template-buffer.h b/framework/src/suricata/src/detect-template-buffer.h new file mode 100644 index 00000000..8a2ab8ba --- /dev/null +++ b/framework/src/suricata/src/detect-template-buffer.h @@ -0,0 +1,25 @@ +/* Copyright (C) 2015 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +#ifndef __DETECT_TEMPLATE_BUFFER_H__ +#define __DETECT_TEMPLATE_BUFFER_H__ + +#include "app-layer-template.h" + +void DetectTemplateBufferRegister(void); + +#endif /* __DETECT_TEMPLATE_BUFFER_H__ */ diff --git a/framework/src/suricata/src/detect.c b/framework/src/suricata/src/detect.c index a1516200..e0b5bfd3 100644 --- a/framework/src/suricata/src/detect.c +++ b/framework/src/suricata/src/detect.c @@ -156,6 +156,7 @@ #include "detect-dns-query.h" #include "detect-app-layer-protocol.h" #include "detect-template.h" +#include "detect-template-buffer.h" #include "util-rule-vars.h" @@ -163,6 +164,7 @@ #include "app-layer-protos.h" #include "app-layer-htp.h" #include "app-layer-smtp.h" +#include "app-layer-template.h" #include "detect-tls.h" #include "detect-tls-version.h" #include "detect-ssh-proto-version.h" @@ -2431,6 +2433,10 @@ PacketCreateMask(Packet *p, SignatureMask *mask, AppProto alproto, int has_state SCLogDebug("packet/flow has smtp state"); (*mask) |= SIG_MASK_REQUIRE_SMTP_STATE; break; + case ALPROTO_TEMPLATE: + SCLogDebug("packet/flow has template state"); + (*mask) |= SIG_MASK_REQUIRE_TEMPLATE_STATE; + break; default: SCLogDebug("packet/flow has other state"); break; @@ -2668,6 +2674,10 @@ static int SignatureCreateMask(Signature *s) s->mask |= SIG_MASK_REQUIRE_SMTP_STATE; SCLogDebug("sig requires smtp state"); } + if (s->alproto == ALPROTO_TEMPLATE) { + s->mask |= SIG_MASK_REQUIRE_TEMPLATE_STATE; + SCLogDebug("sig requires template state"); + } if ((s->mask & SIG_MASK_REQUIRE_DCE_STATE) || (s->mask & SIG_MASK_REQUIRE_HTTP_STATE) || @@ -2675,6 +2685,7 @@ static int SignatureCreateMask(Signature *s) (s->mask & SIG_MASK_REQUIRE_DNS_STATE) || (s->mask & SIG_MASK_REQUIRE_FTP_STATE) || (s->mask & SIG_MASK_REQUIRE_SMTP_STATE) || + (s->mask & SIG_MASK_REQUIRE_TEMPLATE_STATE) || (s->mask & SIG_MASK_REQUIRE_TLS_STATE)) { s->mask |= SIG_MASK_REQUIRE_FLOW; @@ -5238,6 +5249,7 @@ void SigTableSetup(void) DetectModbusRegister(); DetectAppLayerProtocolRegister(); DetectTemplateRegister(); + DetectTemplateBufferRegister(); } void SigTableRegisterTests(void) diff --git a/framework/src/suricata/src/detect.h b/framework/src/suricata/src/detect.h index 236f69aa..30adc9c4 100644 --- a/framework/src/suricata/src/detect.h +++ b/framework/src/suricata/src/detect.h @@ -126,6 +126,8 @@ enum DetectSigmatchListEnum { DETECT_SM_LIST_MODBUS_MATCH, + DETECT_SM_LIST_TEMPLATE_BUFFER_MATCH, + /* list for post match actions: flowbit set, flowint increment, etc */ DETECT_SM_LIST_POSTMATCH, @@ -305,6 +307,7 @@ typedef struct DetectPort_ { #define SIG_MASK_REQUIRE_DNS_STATE (1<<10) #define SIG_MASK_REQUIRE_FTP_STATE (1<<11) #define SIG_MASK_REQUIRE_SMTP_STATE (1<<12) +#define SIG_MASK_REQUIRE_TEMPLATE_STATE (1<<13) /* for now a uint8_t is enough */ #define SignatureMask uint16_t @@ -1229,6 +1232,7 @@ enum { DETECT_XBITS, DETECT_TEMPLATE, + DETECT_AL_TEMPLATE_BUFFER, /* make sure this stays last */ DETECT_TBLSIZE, diff --git a/framework/src/suricata/src/output-json-template.c b/framework/src/suricata/src/output-json-template.c new file mode 100644 index 00000000..ca4a9378 --- /dev/null +++ b/framework/src/suricata/src/output-json-template.c @@ -0,0 +1,213 @@ +/* Copyright (C) 2015 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +#include "suricata-common.h" +#include "debug.h" +#include "detect.h" +#include "pkt-var.h" +#include "conf.h" + +#include "threads.h" +#include "threadvars.h" +#include "tm-threads.h" + +#include "util-unittest.h" +#include "util-buffer.h" +#include "util-debug.h" +#include "util-byte.h" + +#include "output.h" +#include "output-json.h" + +#include "app-layer.h" +#include "app-layer-parser.h" + +#include "app-layer-template.h" + +#ifdef HAVE_LIBJANSSON +#include <jansson.h> + +typedef struct LogTemplateFileCtx_ { + LogFileCtx *file_ctx; + uint32_t flags; +} LogTemplateFileCtx; + +typedef struct LogTemplateLogThread_ { + LogTemplateFileCtx *templatelog_ctx; + uint32_t count; + MemBuffer *buffer; +} LogTemplateLogThread; + +static int JsonTemplateLogger(ThreadVars *tv, void *thread_data, + const Packet *p, Flow *f, void *state, void *tx, uint64_t tx_id) +{ + TemplateTransaction *templatetx = tx; + LogTemplateLogThread *thread = thread_data; + MemBuffer *buffer = thread->buffer; + json_t *js, *templatejs; + + SCLogNotice("Logging template transaction %"PRIu64".", templatetx->tx_id); + + js = CreateJSONHeader((Packet *)p, 0, "template"); + if (unlikely(js == NULL)) { + return TM_ECODE_FAILED; + } + + templatejs = json_object(); + if (unlikely(templatejs == NULL)) { + goto error; + } + + /* Convert the request buffer to a string then log. */ + char *request_buffer = BytesToString(templatetx->request_buffer, + templatetx->request_buffer_len); + if (request_buffer != NULL) { + json_object_set_new(templatejs, "request", json_string(request_buffer)); + SCFree(request_buffer); + } + + /* Convert the response buffer to a string then log. */ + char *response_buffer = BytesToString(templatetx->response_buffer, + templatetx->response_buffer_len); + if (response_buffer != NULL) { + json_object_set_new(templatejs, "response", + json_string(response_buffer)); + SCFree(response_buffer); + } + + json_object_set_new(js, "template", templatejs); + + MemBufferReset(buffer); + OutputJSONBuffer(js, thread->templatelog_ctx->file_ctx, buffer); + + json_decref(js); + return TM_ECODE_OK; + +error: + if (templatejs != NULL) { + json_decref(templatejs); + } + json_decref(js); + return TM_ECODE_FAILED; +} + +static void OutputTemplateLogDeInitCtxSub(OutputCtx *output_ctx) +{ + LogTemplateFileCtx *templatelog_ctx = (LogTemplateFileCtx *)output_ctx->data; + SCFree(templatelog_ctx); + SCFree(output_ctx); +} + +static OutputCtx *OutputTemplateLogInitSub(ConfNode *conf, + OutputCtx *parent_ctx) +{ + AlertJsonThread *ajt = parent_ctx->data; + + LogTemplateFileCtx *templatelog_ctx = SCCalloc(1, sizeof(*templatelog_ctx)); + if (unlikely(templatelog_ctx == NULL)) { + return NULL; + } + templatelog_ctx->file_ctx = ajt->file_ctx; + + OutputCtx *output_ctx = SCCalloc(1, sizeof(*output_ctx)); + if (unlikely(output_ctx == NULL)) { + SCFree(templatelog_ctx); + return NULL; + } + output_ctx->data = templatelog_ctx; + output_ctx->DeInit = OutputTemplateLogDeInitCtxSub; + + SCLogNotice("Template log sub-module initialized."); + + AppLayerParserRegisterLogger(IPPROTO_TCP, ALPROTO_TEMPLATE); + + return output_ctx; +} + +#define OUTPUT_BUFFER_SIZE 65535 + +static TmEcode JsonTemplateLogThreadInit(ThreadVars *t, void *initdata, void **data) +{ + LogTemplateLogThread *thread = SCCalloc(1, sizeof(*thread)); + if (unlikely(thread == NULL)) { + return TM_ECODE_FAILED; + } + + if (initdata == NULL) { + SCLogDebug("Error getting context for Template. \"initdata\" is NULL."); + SCFree(thread); + return TM_ECODE_FAILED; + } + + thread->buffer = MemBufferCreateNew(OUTPUT_BUFFER_SIZE); + if (unlikely(thread->buffer == NULL)) { + SCFree(thread); + return TM_ECODE_FAILED; + } + + thread->templatelog_ctx = ((OutputCtx *)initdata)->data; + *data = (void *)thread; + + return TM_ECODE_OK; +} + +static TmEcode JsonTemplateLogThreadDeinit(ThreadVars *t, void *data) +{ + LogTemplateLogThread *thread = (LogTemplateLogThread *)data; + if (thread == NULL) { + return TM_ECODE_OK; + } + if (thread->buffer != NULL) { + MemBufferFree(thread->buffer); + } + SCFree(thread); + return TM_ECODE_OK; +} + +void TmModuleJsonTemplateLogRegister(void) +{ + tmm_modules[TMM_JSONTEMPLATELOG].name = "JsonTemplateLog"; + tmm_modules[TMM_JSONTEMPLATELOG].ThreadInit = JsonTemplateLogThreadInit; + tmm_modules[TMM_JSONTEMPLATELOG].ThreadDeinit = JsonTemplateLogThreadDeinit; + tmm_modules[TMM_JSONTEMPLATELOG].RegisterTests = NULL; + tmm_modules[TMM_JSONTEMPLATELOG].cap_flags = 0; + tmm_modules[TMM_JSONTEMPLATELOG].flags = TM_FLAG_LOGAPI_TM; + + /* Register as an eve sub-module. */ + OutputRegisterTxSubModule("eve-log", "JsonTemplateLog", "eve-log.template", + OutputTemplateLogInitSub, ALPROTO_TEMPLATE, JsonTemplateLogger); + + SCLogNotice("Template JSON logger registered."); +} + +#else /* No JSON support. */ + +static TmEcode JsonTemplateLogThreadInit(ThreadVars *t, void *initdata, + void **data) +{ + SCLogInfo("Cannot initialize JSON output for template. " + "JSON support was disabled during build."); + return TM_ECODE_FAILED; +} + +void TmModuleJsonTemplateLogRegister(void) +{ + tmm_modules[TMM_JSONTEMPLATELOG].name = "JsonTemplateLog"; + tmm_modules[TMM_JSONTEMPLATELOG].ThreadInit = JsonTemplateLogThreadInit; +} + +#endif /* HAVE_LIBJANSSON */ diff --git a/framework/src/suricata/src/output-json-template.h b/framework/src/suricata/src/output-json-template.h new file mode 100644 index 00000000..d071e182 --- /dev/null +++ b/framework/src/suricata/src/output-json-template.h @@ -0,0 +1,23 @@ +/* Copyright (C) 2015 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +#ifndef __OUTPUT_JSON_TEMPLATE_H__ +#define __OUTPUT_JSON_TEMPLATE_H__ + +void TmModuleJsonTemplateLogRegister(void); + +#endif /* __OUTPUT_JSON_TEMPLATE_H__ */ diff --git a/framework/src/suricata/src/suricata.c b/framework/src/suricata/src/suricata.c index 7c661e19..6c45c57e 100644 --- a/framework/src/suricata/src/suricata.c +++ b/framework/src/suricata/src/suricata.c @@ -102,6 +102,8 @@ #include "output-json.h" +#include "output-json-template.h" + #include "stream-tcp.h" #include "source-nfq.h" @@ -897,6 +899,9 @@ void RegisterAllModules() /* json stats */ TmModuleJsonStatsLogRegister(); + /* Template JSON logger. */ + TmModuleJsonTemplateLogRegister(); + /* log api */ TmModulePacketLoggerRegister(); TmModuleTxLoggerRegister(); diff --git a/framework/src/suricata/src/tm-modules.c b/framework/src/suricata/src/tm-modules.c index 73e9f235..06190b65 100644 --- a/framework/src/suricata/src/tm-modules.c +++ b/framework/src/suricata/src/tm-modules.c @@ -272,6 +272,7 @@ const char * TmModuleTmmIdToString(TmmId id) CASE_CODE (TMM_DETECTLOADER); CASE_CODE (TMM_LUALOG); CASE_CODE (TMM_LOGSTATSLOG); + CASE_CODE (TMM_JSONTEMPLATELOG); CASE_CODE (TMM_RECEIVENETMAP); CASE_CODE (TMM_DECODENETMAP); CASE_CODE (TMM_TLSSTORE); diff --git a/framework/src/suricata/src/tm-threads-common.h b/framework/src/suricata/src/tm-threads-common.h index f6629eaa..6a66b417 100644 --- a/framework/src/suricata/src/tm-threads-common.h +++ b/framework/src/suricata/src/tm-threads-common.h @@ -101,6 +101,7 @@ typedef enum { TMM_JSONFLOWLOG, TMM_JSONNETFLOWLOG, TMM_LOGSTATSLOG, + TMM_JSONTEMPLATELOG, TMM_FLOWMANAGER, TMM_FLOWRECYCLER, |