Added apache 2 license verbage

Change-Id: I6a0b484496f7f3ba77c3584f5ab5a1acf353b298
Signed-off-by: Ashlee Young <ashlee@wildernessvoice.com>

82 files changed, 3123 insertions, 1498 deletions

diff --git a/framework/build/README b/framework/build/README
index 644449fe..f9626d79 100644
--- a/framework/build/README
+++ b/framework/build/README
@@ -1 +1 @@
-Other than this README file, all contents of this folder will be deleted upon each build. While this can be disabled on local systems, it will be overwritten upon a git pull. We will pull source code from ../src and patches from ../patches. These patches will be applied here and the artifacts built up.
+This is where to place any files to patch upstream projects. The patches are whole files and must be added with their full relative path. For example, if you're making applying a patch to onos/apps/vtn/vtnrsc/src/main/java/org/onosproject/vtnrsc, then you'll need that entire directory path here. Use copy and paste of the location after running "pwd". You can then use "mkdir -p" to create the full path.
diff --git a/framework/src/onos/bgp/bgpio/src/main/java/org/onosproject/bgpio/protocol/ver4/BgpPathAttributes.java b/framework/src/onos/bgp/bgpio/src/main/java/org/onosproject/bgpio/protocol/ver4/BgpPathAttributes.java
index 08fea4c6..20a7ba03 100644
--- a/framework/src/onos/bgp/bgpio/src/main/java/org/onosproject/bgpio/protocol/ver4/BgpPathAttributes.java
+++ b/framework/src/onos/bgp/bgpio/src/main/java/org/onosproject/bgpio/protocol/ver4/BgpPathAttributes.java
@@ -28,6 +28,8 @@ import org.onosproject.bgpio.types.LocalPref;
 import org.onosproject.bgpio.types.Med;
 import org.onosproject.bgpio.types.NextHop;
 import org.onosproject.bgpio.types.Origin;
+import org.onosproject.bgpio.types.MpReachNlri;
+import org.onosproject.bgpio.types.MpUnReachNlri;
 import org.onosproject.bgpio.util.UnSupportedAttribute;
 import org.onosproject.bgpio.util.Validation;
 import org.slf4j.Logger;
@@ -127,11 +129,14 @@ public class BgpPathAttributes {
             case LocalPref.LOCAL_PREF_TYPE:
                 pathAttribute = LocalPref.read(cb);
                 break;
-            case MPREACHNLRI_TYPE:
-                //TODO: To be merged later
+            case MpReachNlri.MPREACHNLRI_TYPE:
+                pathAttribute = MpReachNlri.read(cb);
+                isMpReach = ((MpReachNlri) pathAttribute).isMpReachNlriSet();
                 break;
-            case MPUNREACHNLRI_TYPE:
-                //TODO: To be merged later
+            case MpUnReachNlri.MPUNREACHNLRI_TYPE:
+                pathAttribute = MpUnReachNlri.read(cb);
+                isMpUnReach = ((MpUnReachNlri) pathAttribute)
+                        .isMpUnReachNlriSet();
                 break;
             case LINK_STATE_ATTRIBUTE_TYPE:
                 //TODO: To be merged later
diff --git a/framework/src/onos/providers/openflow/flow/src/main/java/org/onosproject/provider/of/flow/impl/FlowEntryBuilder.java b/framework/src/onos/providers/openflow/flow/src/main/java/org/onosproject/provider/of/flow/impl/FlowEntryBuilder.java
index f4ab0173..1354240f 100644
--- a/framework/src/onos/providers/openflow/flow/src/main/java/org/onosproject/provider/of/flow/impl/FlowEntryBuilder.java
+++ b/framework/src/onos/providers/openflow/flow/src/main/java/org/onosproject/provider/of/flow/impl/FlowEntryBuilder.java
@@ -707,12 +707,15 @@ public class FlowEntryBuilder {
                 long tunnelId = match.get(MatchField.TUNNEL_ID).getValue();
                 builder.matchTunnelId(tunnelId);
                 break;
+            case ARP_SHA:
+                mac = MacAddress.valueOf(match.get(MatchField.ARP_SHA).getLong());
+                builder.matchArpSha(mac);
+                break;
             case ARP_THA:
                 mac = MacAddress.valueOf(match.get(MatchField.ARP_THA).getLong());
                 builder.matchArpTha(mac);
                 break;
             case ARP_OP:
-            case ARP_SHA:
             case ARP_SPA:
             case ARP_TPA:
             case MPLS_TC:
diff --git a/framework/src/onos/providers/openflow/flow/src/main/java/org/onosproject/provider/of/flow/impl/FlowModBuilder.java b/framework/src/onos/providers/openflow/flow/src/main/java/org/onosproject/provider/of/flow/impl/FlowModBuilder.java
index 2e5be654..c5de72a8 100644
--- a/framework/src/onos/providers/openflow/flow/src/main/java/org/onosproject/provider/of/flow/impl/FlowModBuilder.java
+++ b/framework/src/onos/providers/openflow/flow/src/main/java/org/onosproject/provider/of/flow/impl/FlowModBuilder.java
@@ -417,13 +417,17 @@ public abstract class FlowModBuilder {
                                   mplsBos.mplsBos() ? OFBooleanValue.TRUE
                                                     : OFBooleanValue.FALSE);
                 break;
+            case ARP_SHA:
+                arpHaCriterion = (ArpHaCriterion) c;
+                mBuilder.setExact(MatchField.ARP_SHA,
+                                  MacAddress.of(arpHaCriterion.mac().toLong()));
+                break;
             case ARP_THA:
                 arpHaCriterion = (ArpHaCriterion) c;
                 mBuilder.setExact(MatchField.ARP_THA,
                                   MacAddress.of(arpHaCriterion.mac().toLong()));
                 break;
             case ARP_OP:
-            case ARP_SHA:
             case ARP_SPA:
             case ARP_TPA:
             case MPLS_TC:
diff --git a/framework/src/suricata/COPYING b/framework/src/suricata/COPYING
index 5b6e7c66..d159169d 100644
--- a/framework/src/suricata/COPYING
+++ b/framework/src/suricata/COPYING
@@ -1,12 +1,12 @@
-		    GNU GENERAL PUBLIC LICENSE
-		       Version 2, June 1991
+                    GNU GENERAL PUBLIC LICENSE
+                       Version 2, June 1991
 
- Copyright (C) 1989, 1991 Free Software Foundation, Inc.
-                       59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  Everyone is permitted to copy and distribute verbatim copies
  of this license document, but changing it is not allowed.
 
-			    Preamble
+                            Preamble
 
   The licenses for most software are designed to take away your
 freedom to share and change it.  By contrast, the GNU General Public
@@ -15,7 +15,7 @@ software--to make sure the software is free for all its users.  This
 General Public License applies to most of the Free Software
 Foundation's software and to any other program whose authors commit to
 using it.  (Some other Free Software Foundation software is covered by
-the GNU Library General Public License instead.)  You can apply it to
+the GNU Lesser General Public License instead.)  You can apply it to
 your programs, too.
 
   When we speak of free software, we are referring to freedom, not
@@ -55,8 +55,8 @@ patent must be licensed for everyone's free use or not licensed at all.
 
   The precise terms and conditions for copying, distribution and
 modification follow.
-
-		    GNU GENERAL PUBLIC LICENSE
+
+                    GNU GENERAL PUBLIC LICENSE
    TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
 
   0. This License applies to any program or other work which contains
@@ -110,7 +110,7 @@ above, provided that you also meet all of these conditions:
     License.  (Exception: if the Program itself is interactive but
     does not normally print such an announcement, your work based on
     the Program is not required to print an announcement.)
-
+
 These requirements apply to the modified work as a whole.  If
 identifiable sections of that work are not derived from the Program,
 and can be reasonably considered independent and separate works in
@@ -168,7 +168,7 @@ access to copy from a designated place, then offering equivalent
 access to copy the source code from the same place counts as
 distribution of the source code, even though third parties are not
 compelled to copy the source along with the object code.
-
+
   4. You may not copy, modify, sublicense, or distribute the Program
 except as expressly provided under this License.  Any attempt
 otherwise to copy, modify, sublicense or distribute the Program is
@@ -225,7 +225,7 @@ impose that choice.
 
 This section is intended to make thoroughly clear what is believed to
 be a consequence of the rest of this License.
-
+
   8. If the distribution and/or use of the Program is restricted in
 certain countries either by patents or by copyrighted interfaces, the
 original copyright holder who places the Program under this License
@@ -255,7 +255,7 @@ make exceptions for this.  Our decision will be guided by the two goals
 of preserving the free status of all derivatives of our free software and
 of promoting the sharing and reuse of software generally.
 
-			    NO WARRANTY
+                            NO WARRANTY
 
   11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
 FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
@@ -277,9 +277,9 @@ YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
 PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
 POSSIBILITY OF SUCH DAMAGES.
 
-		     END OF TERMS AND CONDITIONS
-
-	    How to Apply These Terms to Your New Programs
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
 
   If you develop a new program, and you want it to be of the greatest
 possible use to the public, the best way to achieve this is to make it
@@ -303,10 +303,9 @@ the "copyright" line and a pointer to where the full notice is found.
     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
     GNU General Public License for more details.
 
-    You should have received a copy of the GNU General Public License
-    along with this program; if not, write to the Free Software
-    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 Also add information on how to contact you by electronic and paper mail.
 
@@ -336,5 +335,5 @@ necessary.  Here is a sample; alter the names:
 This General Public License does not permit incorporating your program into
 proprietary programs.  If your program is a subroutine library, you may
 consider it more useful to permit linking proprietary applications with the
-library.  If this is what you want to do, use the GNU Library General
+library.  If this is what you want to do, use the GNU Lesser General
 Public License instead of this License.
diff --git a/framework/src/suricata/LICENSE b/framework/src/suricata/LICENSE
index d511905c..d159169d 100644
--- a/framework/src/suricata/LICENSE
+++ b/framework/src/suricata/LICENSE
@@ -1,12 +1,12 @@
-		    GNU GENERAL PUBLIC LICENSE
-		       Version 2, June 1991
+                    GNU GENERAL PUBLIC LICENSE
+                       Version 2, June 1991
 
  Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  Everyone is permitted to copy and distribute verbatim copies
  of this license document, but changing it is not allowed.
 
-			    Preamble
+                            Preamble
 
   The licenses for most software are designed to take away your
 freedom to share and change it.  By contrast, the GNU General Public
@@ -56,7 +56,7 @@ patent must be licensed for everyone's free use or not licensed at all.
   The precise terms and conditions for copying, distribution and
 modification follow.
 
-		    GNU GENERAL PUBLIC LICENSE
+                    GNU GENERAL PUBLIC LICENSE
    TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
 
   0. This License applies to any program or other work which contains
@@ -255,7 +255,7 @@ make exceptions for this.  Our decision will be guided by the two goals
 of preserving the free status of all derivatives of our free software and
 of promoting the sharing and reuse of software generally.
 
-			    NO WARRANTY
+                            NO WARRANTY
 
   11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
 FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
@@ -277,9 +277,9 @@ YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
 PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
 POSSIBILITY OF SUCH DAMAGES.
 
-		     END OF TERMS AND CONDITIONS
+                     END OF TERMS AND CONDITIONS
 
-	    How to Apply These Terms to Your New Programs
+            How to Apply These Terms to Your New Programs
 
   If you develop a new program, and you want it to be of the greatest
 possible use to the public, the best way to achieve this is to make it
diff --git a/framework/src/suricata/configure.ac b/framework/src/suricata/configure.ac
index ea819f4a..9e139383 100644
--- a/framework/src/suricata/configure.ac
+++ b/framework/src/suricata/configure.ac
@@ -1113,6 +1113,12 @@
     AC_ARG_ENABLE(non-bundled-htp,
            AS_HELP_STRING([--enable-non-bundled-htp], [Enable the use of an already installed version of htp]),,[enable_non_bundled_htp=no])
     AS_IF([test "x$enable_non_bundled_htp" = "xyes"], [
+        PKG_CHECK_MODULES([libhtp], htp,, [with_pkgconfig_htp=no])
+        if test "$with_pkgconfig_htp" != "no"; then
+            CPPFLAGS="${CPPFLAGS} ${libhtp_CFLAGS}"
+            LIBS="${LIBS} ${libhtp_LIBS}"
+        fi
+
         AC_ARG_WITH(libhtp_includes,
                 [  --with-libhtp-includes=DIR  libhtp include directory],
                 [with_libhtp_includes="$withval"],[with_libhtp_includes=no])
@@ -1687,6 +1693,56 @@
         enable_geoip="yes"
     fi
 
+  # Position Independent Executable
+    AC_ARG_ENABLE(pie,
+                AS_HELP_STRING([--enable-pie],[Enable compiling as a position independent executable]),
+                [ enable_pie="yes"],
+                [ enable_pie="no"])
+    if test "$enable_pie" = "yes"; then
+        CPPFLAGS="${CPPFLAGS} -fPIC"
+        LDFLAGS="${LDFLAGS} -pie"
+    fi
+
+# libhiredis
+    AC_ARG_ENABLE(hiredis,
+	        AS_HELP_STRING([--enable-hiredis],[Enable Redis support]),
+	        [ enable_hiredis="yes"],
+	        [ enable_hiredis="no"])
+    AC_ARG_WITH(libhiredis_includes,
+            [  --with-libhiredis-includes=DIR  libhiredis include directory],
+            [with_libhiredis_includes="$withval"],[with_libhiredis_includes="no"])
+    AC_ARG_WITH(libhiredis_libraries,
+            [  --with-libhiredis-libraries=DIR    libhiredis library directory],
+            [with_libhiredis_libraries="$withval"],[with_libhiredis_libraries="no"])
+
+    if test "$enable_hiredis" = "yes"; then
+        if test "$with_libhiredis_includes" != "no"; then
+            CPPFLAGS="${CPPFLAGS} -I${with_libhiredis_includes}"
+        fi
+
+        AC_CHECK_HEADER("hiredis/hiredis.h",HIREDIS="yes",HIREDIS="no")
+        if test "$HIREDIS" = "yes"; then
+            if test "$with_libhiredis_libraries" != "no"; then
+                LDFLAGS="${LDFLAGS}  -L${with_libhiredis_libraries}"
+            fi
+            AC_CHECK_LIB(hiredis, redisConnect,, HIREDIS="no")
+        fi
+        if test "$HIREDIS" = "no"; then
+            echo
+            echo "   ERROR!  libhiredis library not found, go get it"
+            echo "   from https://github.com/redis/hiredis or your distribution:"
+            echo
+            echo "   Ubuntu: apt-get install libhiredis-dev"
+            echo "   Fedora: yum install libhiredis-devel"
+            echo
+            exit 1
+        fi
+        if test "$HIREDIS" = "yes"; then
+            AC_DEFINE([HAVE_LIBHIREDIS],[1],[libhiredis available])
+            enable_hiredis="yes"
+        fi
+    fi
+
 # get cache line size
     AC_PATH_PROG(HAVE_GETCONF_CMD, getconf, "no")
     if test "$HAVE_GETCONF_CMD" != "no"; then
@@ -1794,6 +1850,7 @@ SURICATA_BUILD_CONF="Suricata Configuration:
   libnss support:                          ${enable_nss}
   libnspr support:                         ${enable_nspr}
   libjansson support:                      ${enable_jansson}
+  hiredis support:                         ${enable_hiredis}
   Prelude support:                         ${enable_prelude}
   PCRE jit:                                ${pcre_jit_available}
   LUA support:                             ${enable_lua}
@@ -1826,6 +1883,7 @@ Generic build parameters:
   GCC Protect enabled:                     ${enable_gccprotect}
   GCC march native enabled:                ${enable_gccmarch_native}
   GCC Profile enabled:                     ${enable_gccprofile}
+  Position Independent Executable enabled: ${enable_pie}
   CFLAGS                                   ${CFLAGS}
   PCAP_CFLAGS                              ${PCAP_CFLAGS}
   SECCFLAGS                                ${SECCFLAGS}"
diff --git a/framework/src/suricata/qa/prscript.py b/framework/src/suricata/qa/prscript.py
index 4f401edd..02dd0ad9 100755
--- a/framework/src/suricata/qa/prscript.py
+++ b/framework/src/suricata/qa/prscript.py
@@ -72,6 +72,7 @@ parser.add_argument('-d', '--docker', action='store_const', const=True, help='us
 parser.add_argument('-C', '--create', action='store_const', const=True, help='create docker container' + docker_deps, default=False)
 parser.add_argument('-s', '--start', action='store_const', const=True, help='start docker container' + docker_deps, default=False)
 parser.add_argument('-S', '--stop', action='store_const', const=True, help='stop docker container' + docker_deps, default=False)
+parser.add_argument('-R', '--rm', action='store_const', const=True, help='remove docker container and image' + docker_deps, default=False)
 parser.add_argument('branch', metavar='branch', help='github branch to build', nargs='?')
 args = parser.parse_args()
 username = args.username
@@ -227,9 +228,6 @@ if not args.local and TestRepoSync(args.branch) == -1:
         sys.exit(-1)
 
 def CreateContainer():
-    if not os.geteuid() == 0:
-        print "Command must be run as root"
-        sys.exit(-1)
     cli = Client()
     # FIXME check if existing
     print "Pulling docking image, first run should take long"
@@ -238,9 +236,6 @@ def CreateContainer():
     sys.exit(0)
 
 def StartContainer():
-    if not os.geteuid() == 0:
-        print "Command must be run as root"
-        sys.exit(-1)
     cli = Client()
     suri_src_dir = os.path.split(os.path.dirname(os.path.realpath(__file__)))[0]
     print "Using base src dir: " + suri_src_dir
@@ -248,13 +243,24 @@ def StartContainer():
     sys.exit(0)
 
 def StopContainer():
-    if not os.geteuid() == 0:
-        print "Command must be run as root"
-        sys.exit(-1)
     cli = Client()
     cli.stop('suri-buildbot')
     sys.exit(0)
 
+def RmContainer():
+    cli = Client()
+    try:
+        cli.remove_container('suri-buildbot')
+    except:
+        print "Unable to remove suri-buildbot container"
+        pass
+    try:
+        cli.remove_image('regit/suri-buildbot:latest')
+    except:
+        print "Unable to remove suri-buildbot images"
+        pass
+    sys.exit(0)
+
 if GOT_DOCKER:
     if args.create:
         CreateContainer()
@@ -262,6 +268,8 @@ if GOT_DOCKER:
         StartContainer()
     if args.stop:
         StopContainer()
+    if args.rm:
+        RmContainer()
 
 if not args.branch:
     print "You need to specify a branch for this mode"
diff --git a/framework/src/suricata/scripts/setup-app-layer-detect.sh b/framework/src/suricata/scripts/setup-app-layer-detect.sh
index ef3b741e..ef4bccaa 100755
--- a/framework/src/suricata/scripts/setup-app-layer-detect.sh
+++ b/framework/src/suricata/scripts/setup-app-layer-detect.sh
@@ -34,9 +34,11 @@ function copy_template_file() {
 
     echo "Creating ${dst}."
 
-    sed -e "s/TEMPLATE/${protoname_upper}/g" \
+    sed -e '/TEMPLATE_START_REMOVE/,/TEMPLATE_END_REMOVE/d' \
+	-e "s/TEMPLATE/${protoname_upper}/g" \
 	-e "s/template/${protoname_lower}/g" \
-	-e "s/Template/${protoname}/g" > ${dst} < ${src}
+	-e "s/Template/${protoname}/g" \
+	> ${dst} < ${src}
 }
 
 function copy_templates() {
diff --git a/framework/src/suricata/scripts/setup-app-layer-logger.sh b/framework/src/suricata/scripts/setup-app-layer-logger.sh
index be32c393..13342412 100755
--- a/framework/src/suricata/scripts/setup-app-layer-logger.sh
+++ b/framework/src/suricata/scripts/setup-app-layer-logger.sh
@@ -38,9 +38,11 @@ function copy_template_file() {
 
     echo "Creating ${dst}."
     
-    sed -e "s/TEMPLATE/${protoname_upper}/g" \
+    sed -e '/TEMPLATE_START_REMOVE/,/TEMPLATE_END_REMOVE/d' \
+	-e "s/TEMPLATE/${protoname_upper}/g" \
 	-e "s/template/${protoname_lower}/g" \
-	-e "s/Template/${protoname}/g" > ${dst} < ${src}
+	-e "s/Template/${protoname}/g" \
+	> ${dst} < ${src}
 }
 
 function copy_templates() {
@@ -62,7 +64,7 @@ function patch_makefile_am() {
     ed -s ${filename} > /dev/null <<EOF
 /output-json-template.c
 t-
-s/template/${protoname_lower}/
+s/template/${protoname_lower}/g
 w
 EOF
 }
diff --git a/framework/src/suricata/scripts/setup-app-layer.sh b/framework/src/suricata/scripts/setup-app-layer.sh
index b24b5e61..2789f20d 100755
--- a/framework/src/suricata/scripts/setup-app-layer.sh
+++ b/framework/src/suricata/scripts/setup-app-layer.sh
@@ -37,9 +37,11 @@ function copy_template_file() {
 
     echo "Creating ${dst}."
     
-    sed -e "s/TEMPLATE/${protoname_upper}/g" \
+    sed -e '/TEMPLATE_START_REMOVE/,/TEMPLATE_END_REMOVE/d' \
+	-e "s/TEMPLATE/${protoname_upper}/g" \
 	-e "s/template/${protoname_lower}/g" \
-	-e "s/Template/${protoname}/g" > ${dst} < ${src}
+	-e "s/Template/${protoname}/g" \
+	> ${dst} < ${src}
 }
 
 function copy_app_layer_templates {
diff --git a/framework/src/suricata/src/Makefile.am b/framework/src/suricata/src/Makefile.am
index 674043f5..98e094c1 100644
--- a/framework/src/suricata/src/Makefile.am
+++ b/framework/src/suricata/src/Makefile.am
@@ -356,6 +356,7 @@ util-lua-common.c util-lua-common.h \
 util-lua-dns.c util-lua-dns.h \
 util-lua-http.c util-lua-http.h \
 util-lua-tls.c util-lua-tls.h \
+util-lua-ssh.c util-lua-ssh.h \
 util-magic.c util-magic.h \
 util-memcmp.c util-memcmp.h \
 util-memcpy.h \
diff --git a/framework/src/suricata/src/alert-unified2-alert.c b/framework/src/suricata/src/alert-unified2-alert.c
index ede624c4..facc66b2 100644
--- a/framework/src/suricata/src/alert-unified2-alert.c
+++ b/framework/src/suricata/src/alert-unified2-alert.c
@@ -186,8 +186,11 @@ typedef struct AlertUnified2Packet_ {
 typedef struct Unified2AlertFileCtx_ {
     LogFileCtx *file_ctx;
     HttpXFFCfg *xff_cfg;
+    uint32_t flags; /**< flags for all alerts */
 } Unified2AlertFileCtx;
 
+#define UNIFIED2_ALERT_FLAGS_EMIT_PACKET (1 << 0)
+
 /**
  * Unified2 thread vars
  *
@@ -698,6 +701,9 @@ static int Unified2PacketTypeAlert(Unified2AlertThread *aun, const Packet *p, ui
 {
     int ret = 0;
 
+    if (!(aun->unified2alert_ctx->flags & UNIFIED2_ALERT_FLAGS_EMIT_PACKET))
+        return 1;
+
     /* try stream logging first */
     if (stream) {
         SCLogDebug("logging the state");
@@ -1299,6 +1305,20 @@ OutputCtx *Unified2AlertInitCtx(ConfNode *conf)
         }
     }
 
+    uint32_t flags = UNIFIED2_ALERT_FLAGS_EMIT_PACKET;
+    if (conf != NULL) {
+        const char *payload = NULL;
+        payload = ConfNodeLookupChildValue(conf, "payload");
+        if (payload) {
+            if (ConfValIsFalse(payload)) {
+                flags &= ~UNIFIED2_ALERT_FLAGS_EMIT_PACKET;
+            } else if (!ConfValIsTrue(payload)) {
+                SCLogError(SC_ERR_INVALID_ARGUMENT, "Failed to initialize unified2 output, invalid payload: %s", payload);
+                exit(EXIT_FAILURE);
+            }
+        }
+    }
+
     ret = Unified2AlertOpenFileCtx(file_ctx, filename);
     if (ret < 0)
         goto error;
@@ -1325,6 +1345,7 @@ OutputCtx *Unified2AlertInitCtx(ConfNode *conf)
 
     unified2alert_ctx->file_ctx = file_ctx;
     unified2alert_ctx->xff_cfg = xff_cfg;
+    unified2alert_ctx->flags = flags;
     output_ctx->data = unified2alert_ctx;
     output_ctx->DeInit = Unified2AlertDeInitCtx;
 
diff --git a/framework/src/suricata/src/app-layer-htp.c b/framework/src/suricata/src/app-layer-htp.c
index 452a607e..0abcda3b 100644
--- a/framework/src/suricata/src/app-layer-htp.c
+++ b/framework/src/suricata/src/app-layer-htp.c
@@ -78,10 +78,6 @@
 
 #include "util-memcmp.h"
 
-#ifndef HAVE_HTP_SET_PATH_DECODE_U_ENCODING
-void htp_config_set_path_decode_u_encoding(htp_cfg_t *cfg, int decode_u_encoding);
-#endif
-
 //#define PRINT
 
 /** Fast lookup tree (radix) for the various HTP configurations */
diff --git a/framework/src/suricata/src/app-layer-smtp.c b/framework/src/suricata/src/app-layer-smtp.c
index 1951613d..0c161edb 100644
--- a/framework/src/suricata/src/app-layer-smtp.c
+++ b/framework/src/suricata/src/app-layer-smtp.c
@@ -220,7 +220,9 @@ SCEnumCharMap smtp_reply_map[ ] = {
 };
 
 /* Create SMTP config structure */
-SMTPConfig smtp_config = { 0, { 0, 0, 0, 0 }, 0, 0, 0};
+SMTPConfig smtp_config = { 0, { 0, 0, 0, 0, 0 }, 0, 0, 0};
+
+static SMTPString *SMTPStringAlloc(void);
 
 /**
  * \brief Configure SMTP Mime Decoder by parsing out mime section of YAML
@@ -264,6 +266,11 @@ static void SMTPConfigure(void) {
         if (ret) {
             smtp_config.mime_config.extract_urls = val;
         }
+
+        ret = ConfGetChildValueBool(config, "body-md5", &val);
+        if (ret) {
+            smtp_config.mime_config.body_md5 = val;
+        }
     }
 
     /* Pass mime config data to MimeDec API */
@@ -323,6 +330,7 @@ static SMTPTransaction *SMTPTransactionCreate(void)
         return NULL;
     }
 
+    TAILQ_INIT(&tx->rcpt_to_list);
     tx->mime_state = NULL;
     return tx;
 }
@@ -366,7 +374,7 @@ int SMTPProcessDataChunk(const uint8_t *chunk, uint32_t len,
             if (smtp_state->files_ts == NULL) {
                 ret = MIME_DEC_ERR_MEM;
                 SCLogError(SC_ERR_MEM_ALLOC, "Could not create file container");
-                goto end;
+                SCReturnInt(ret);
             }
         }
         files = smtp_state->files_ts;
@@ -448,7 +456,7 @@ int SMTPProcessDataChunk(const uint8_t *chunk, uint32_t len,
     if (files != NULL) {
         FilePrune(files);
     }
-end:
+
     SCReturnInt(ret);
 }
 
@@ -805,7 +813,8 @@ static int SMTPProcessCommandDATA(SMTPState *state, Flow *f,
 
         if (smtp_config.decode_mime && state->curr_tx->mime_state) {
             int ret = MimeDecParseLine((const uint8_t *) state->current_line,
-                    state->current_line_len, state->curr_tx->mime_state);
+                    state->current_line_len, state->current_line_delimiter_len,
+                    state->curr_tx->mime_state);
             if (ret != MIME_DEC_OK) {
                 SCLogDebug("MimeDecParseLine() function returned an error code: %d", ret);
             }
@@ -955,6 +964,71 @@ static int SMTPParseCommandBDAT(SMTPState *state)
     return 0;
 }
 
+static int SMTPParseCommandWithParam(SMTPState *state, uint8_t prefix_len, uint8_t **target, uint16_t *target_len)
+{
+    int i = prefix_len + 1;
+    int spc_i = 0;
+
+    while (i < state->current_line_len) {
+        if (state->current_line[i] != ' ') {
+            break;
+        }
+        i++;
+    }
+
+    /* rfc1870: with the size extension the mail from can be followed by an option.
+       We use the space separator to detect it. */
+    spc_i = i;
+    while (spc_i < state->current_line_len) {
+        if (state->current_line[spc_i] == ' ') {
+            break;
+        }
+        spc_i++;
+    }
+
+    *target = SCMalloc(spc_i - i + 1);
+    if (*target == NULL)
+        return -1;
+    memcpy(*target, state->current_line + i, spc_i - i);
+    (*target)[spc_i - i] = '\0';
+    *target_len = spc_i - i;
+
+    return 0;
+}
+
+static int SMTPParseCommandHELO(SMTPState *state)
+{
+    return SMTPParseCommandWithParam(state, 4, &state->helo, &state->helo_len);
+}
+
+static int SMTPParseCommandMAILFROM(SMTPState *state)
+{
+    return SMTPParseCommandWithParam(state, 9,
+                                     &state->curr_tx->mail_from,
+                                     &state->curr_tx->mail_from_len);
+}
+
+static int SMTPParseCommandRCPTTO(SMTPState *state)
+{
+    uint8_t *rcptto;
+    uint16_t rcptto_len;
+
+    if (SMTPParseCommandWithParam(state, 7, &rcptto, &rcptto_len) == 0) {
+        SMTPString *rcptto_str = SMTPStringAlloc();
+        if (rcptto_str) {
+            rcptto_str->str = rcptto;
+            rcptto_str->len = rcptto_len;
+            TAILQ_INSERT_TAIL(&state->curr_tx->rcpt_to_list, rcptto_str, next);
+        } else {
+            SCFree(rcptto);
+            return -1;
+        }
+    } else {
+        return -1;
+    }
+    return 0;
+}
+
 /* consider 'rset' and 'quit' to be part of the existing state */
 static int NoNewTx(SMTPState *state)
 {
@@ -1027,6 +1101,28 @@ static int SMTPProcessRequest(SMTPState *state, Flow *f,
             }
             state->current_command = SMTP_COMMAND_BDAT;
             state->parser_state |= SMTP_PARSER_STATE_COMMAND_DATA_MODE;
+        } else if (state->current_line_len >= 4 &&
+                   ((SCMemcmpLowercase("helo", state->current_line, 4) == 0) ||
+                    SCMemcmpLowercase("ehlo", state->current_line, 4) == 0))  {
+            r = SMTPParseCommandHELO(state);
+            if (r == -1) {
+                SCReturnInt(-1);
+            }
+            state->current_command = SMTP_COMMAND_OTHER_CMD;
+        } else if (state->current_line_len >= 9 &&
+                   SCMemcmpLowercase("mail from", state->current_line, 9) == 0) {
+            r = SMTPParseCommandMAILFROM(state);
+            if (r == -1) {
+                SCReturnInt(-1);
+            }
+            state->current_command = SMTP_COMMAND_OTHER_CMD;
+        } else if (state->current_line_len >= 7 &&
+                   SCMemcmpLowercase("rcpt to", state->current_line, 7) == 0) {
+            r = SMTPParseCommandRCPTTO(state);
+            if (r == -1) {
+                SCReturnInt(-1);
+            }
+            state->current_command = SMTP_COMMAND_OTHER_CMD;
         } else {
             state->current_command = SMTP_COMMAND_OTHER_CMD;
         }
@@ -1142,6 +1238,25 @@ void *SMTPStateAlloc(void)
     return smtp_state;
 }
 
+static SMTPString *SMTPStringAlloc(void)
+{
+    SMTPString *smtp_string = SCMalloc(sizeof(SMTPString));
+    if (unlikely(smtp_string == NULL))
+        return NULL;
+    memset(smtp_string, 0, sizeof(SMTPString));
+
+    return smtp_string;
+}
+
+
+static void SMTPStringFree(SMTPString *str)
+{
+    if (str->str) {
+        SCFree(str->str);
+    }
+    SCFree(str);
+}
+
 static void *SMTPLocalStorageAlloc(void)
 {
     /* needed by the mpm */
@@ -1178,6 +1293,15 @@ static void SMTPTransactionFree(SMTPTransaction *tx, SMTPState *state)
 
     if (tx->de_state != NULL)
         DetectEngineStateFree(tx->de_state);
+
+    if (tx->mail_from)
+        SCFree(tx->mail_from);
+
+    SMTPString *str = NULL;
+    while ((str = TAILQ_FIRST(&tx->rcpt_to_list))) {
+        TAILQ_REMOVE(&tx->rcpt_to_list, str, next);
+        SMTPStringFree(str);
+    }
 #if 0
         if (tx->decoder_events->cnt <= smtp_state->events)
             smtp_state->events -= tx->decoder_events->cnt;
@@ -1205,6 +1329,10 @@ static void SMTPStateFree(void *p)
         SCFree(smtp_state->tc_db);
     }
 
+    if (smtp_state->helo) {
+        SCFree(smtp_state->helo);
+    }
+
     FileContainerFree(smtp_state->files_ts);
 
     SMTPTransaction *tx = NULL;
@@ -4365,6 +4493,13 @@ int SMTPParserTest14(void)
         SCMutexUnlock(&f.m);
         goto end;
     }
+
+    if ((smtp_state->helo_len != 7) || strncmp("boo.com", (char *)smtp_state->helo, 7)) {
+        printf("incorrect parsing of HELO field '%s' (%d)\n", smtp_state->helo, smtp_state->helo_len);
+        SCMutexUnlock(&f.m);
+        goto end;
+    }
+
     SCMutexUnlock(&f.m);
     if (smtp_state->input_len != 0 ||
             smtp_state->cmds_cnt != 0 ||
@@ -4402,6 +4537,16 @@ int SMTPParserTest14(void)
         SCMutexUnlock(&f.m);
         goto end;
     }
+
+    if ((smtp_state->curr_tx->mail_from_len != 14) ||
+        strncmp("asdff@asdf.com", (char *)smtp_state->curr_tx->mail_from, 14)) {
+        printf("incorrect parsing of MAIL FROM field '%s' (%d)\n",
+               smtp_state->curr_tx->mail_from,
+               smtp_state->curr_tx->mail_from_len);
+        SCMutexUnlock(&f.m);
+        goto end;
+    }
+
     SCMutexUnlock(&f.m);
     if (smtp_state->input_len != 0 ||
             smtp_state->cmds_cnt != 0 ||
diff --git a/framework/src/suricata/src/app-layer-smtp.h b/framework/src/suricata/src/app-layer-smtp.h
index 02090c61..c5868414 100644
--- a/framework/src/suricata/src/app-layer-smtp.h
+++ b/framework/src/suricata/src/app-layer-smtp.h
@@ -51,6 +51,13 @@ enum {
     SMTP_DECODER_EVENT_MIME_BOUNDARY_TOO_LONG,
 };
 
+typedef struct SMTPString_ {
+    uint8_t *str;
+    uint16_t len;
+
+    TAILQ_ENTRY(SMTPString_) next;
+} SMTPString;
+
 typedef struct SMTPTransaction_ {
     /** id of this tx, starting at 0 */
     uint64_t tx_id;
@@ -65,6 +72,12 @@ typedef struct SMTPTransaction_ {
     AppLayerDecoderEvents *decoder_events;          /**< per tx events */
     DetectEngineState *de_state;
 
+    /* MAIL FROM parameters */
+    uint8_t *mail_from;
+    uint16_t mail_from_len;
+
+    TAILQ_HEAD(, SMTPString_) rcpt_to_list;  /**< rcpt to string list */
+
     TAILQ_ENTRY(SMTPTransaction_) next;
 } SMTPTransaction;
 
@@ -136,6 +149,9 @@ typedef struct SMTPState_ {
     /** the list of files sent to the server */
     FileContainer *files_ts;
 
+    /* HELO of HELO message content */
+    uint8_t *helo;
+    uint16_t helo_len;
 } SMTPState;
 
 /* Create SMTP config structure */
diff --git a/framework/src/suricata/src/app-layer-ssh.h b/framework/src/suricata/src/app-layer-ssh.h
index 7a6a9b72..4fd59aa9 100644
--- a/framework/src/suricata/src/app-layer-ssh.h
+++ b/framework/src/suricata/src/app-layer-ssh.h
@@ -34,6 +34,8 @@
 
 #define SSH_FLAG_STATE_LOGGED                0x04
 
+#define SSH_FLAG_STATE_LOGGED_LUA            0x08
+
 /* MSG_CODE */
 #define SSH_MSG_NEWKEYS                      21
 
diff --git a/framework/src/suricata/src/app-layer-ssl.h b/framework/src/suricata/src/app-layer-ssl.h
index 5c6fe5b1..2fc1a969 100644
--- a/framework/src/suricata/src/app-layer-ssl.h
+++ b/framework/src/suricata/src/app-layer-ssl.h
@@ -81,6 +81,8 @@ enum {
 /* flags for file storage */
 #define SSL_AL_FLAG_STATE_STORED                0x40000
 
+#define SSL_AL_FLAG_STATE_LOGGED_LUA            0x80000
+
 /* config flags */
 #define SSL_TLS_LOG_PEM                         (1 << 0)
 
diff --git a/framework/src/suricata/src/app-layer-template.c b/framework/src/suricata/src/app-layer-template.c
index 87b64b73..5eafa67e 100644
--- a/framework/src/suricata/src/app-layer-template.c
+++ b/framework/src/suricata/src/app-layer-template.c
@@ -25,6 +25,7 @@
 
 #include "suricata-common.h"
 #include "stream.h"
+#include "conf.h"
 
 #include "util-unittest.h"
 
@@ -428,6 +429,12 @@ void RegisterTemplateParsers(void)
 {
     char *proto_name = "template";
 
+    /* TEMPLATE_START_REMOVE */
+    if (ConfGetNode("app-layer.protocols.template") == NULL) {
+        return;
+    }
+    /* TEMPLATE_END_REMOVE */
+
     /* Check if Template TCP detection is enabled. If it does not exist in
      * the configuration file then it will be enabled by default. */
     if (AppLayerProtoDetectConfProtoDetectionEnabled("tcp", proto_name)) {
@@ -481,8 +488,8 @@ void RegisterTemplateParsers(void)
             STREAM_TOSERVER, TemplateParseRequest);
 
         /* Register response parser for parsing frames from server to client. */
-        AppLayerParserRegisterParser(IPPROTO_TCP, ALPROTO_TEMPLATE, STREAM_TOCLIENT,
-            TemplateParseResponse);
+        AppLayerParserRegisterParser(IPPROTO_TCP, ALPROTO_TEMPLATE,
+            STREAM_TOCLIENT, TemplateParseResponse);
 
         /* Register a function to be called by the application layer
          * when a transaction is to be freed. */
@@ -490,7 +497,8 @@ void RegisterTemplateParsers(void)
             TemplateStateTxFree);
 
         /* Register a function to return the current transaction count. */
-        AppLayerParserRegisterGetTxCnt(IPPROTO_TCP, ALPROTO_TEMPLATE, TemplateGetTxCnt);
+        AppLayerParserRegisterGetTxCnt(IPPROTO_TCP, ALPROTO_TEMPLATE,
+            TemplateGetTxCnt);
 
         /* Transaction handling. */
         AppLayerParserRegisterGetStateProgressCompletionStatus(IPPROTO_TCP,
diff --git a/framework/src/suricata/src/app-layer.c b/framework/src/suricata/src/app-layer.c
index c47c2dd6..96fa252c 100644
--- a/framework/src/suricata/src/app-layer.c
+++ b/framework/src/suricata/src/app-layer.c
@@ -74,6 +74,12 @@ static void DisableAppLayer(Flow *f)
     StreamTcpDisableAppLayer(f);
 }
 
+static inline int ProtoDetectDone(const Flow *f, const TcpSession *ssn, uint8_t direction) {
+    const TcpStream *stream = (direction & STREAM_TOSERVER) ? &ssn->client : &ssn->server;
+    return ((stream->flags & STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_COMPLETED) ||
+            (FLOW_IS_PM_DONE(f, direction) && FLOW_IS_PP_DONE(f, direction)));
+}
+
 int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
                           Packet *p, Flow *f,
                           TcpSession *ssn, TcpStream *stream,
@@ -187,7 +193,14 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
                         p->flowflags |= FLOW_PKT_TOCLIENT;
                     }
                 }
-                int ret = StreamTcpReassembleAppLayer(tv, ra_ctx, ssn,
+
+                int ret = 0;
+                /* if the opposing side is not going to work, then
+                 * we just have to give up. */
+                if (opposing_stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY)
+                    ret = -1;
+                else
+                    ret = StreamTcpReassembleAppLayer(tv, ra_ctx, ssn,
                                                       opposing_stream, p);
                 if (stream == &ssn->client) {
                     if (StreamTcpInlineMode()) {
@@ -329,7 +342,7 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
                 } else {
                     f->data_al_so_far[dir] = data_len;
                 }
-            } else {
+             } else {
                 /* See if we're going to have to give up:
                  *
                  * If we're getting a lot of data in one direction and the
@@ -349,9 +362,16 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
                 uint32_t size_ts = ssn->client.last_ack - ssn->client.isn - 1;
                 uint32_t size_tc = ssn->server.last_ack - ssn->server.isn - 1;
                 SCLogDebug("size_ts %u, size_tc %u", size_ts, size_tc);
-
-                if (FLOW_IS_PM_DONE(f, STREAM_TOSERVER) && FLOW_IS_PP_DONE(f, STREAM_TOSERVER) &&
-                    FLOW_IS_PM_DONE(f, STREAM_TOCLIENT) && FLOW_IS_PP_DONE(f, STREAM_TOCLIENT)) {
+#ifdef DEBUG_VALIDATION
+                if (!(ssn->client.flags & STREAMTCP_STREAM_FLAG_GAP))
+                    BUG_ON(size_ts > 1000000UL);
+                if (!(ssn->server.flags & STREAMTCP_STREAM_FLAG_GAP))
+                    BUG_ON(size_tc > 1000000UL);
+#endif /* DEBUG_VALIDATION */
+
+                if (ProtoDetectDone(f, ssn, STREAM_TOSERVER) &&
+                    ProtoDetectDone(f, ssn, STREAM_TOCLIENT))
+                {
                     DisableAppLayer(f);
                     ssn->data_first_seen_dir = APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER;
 
@@ -391,8 +411,19 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
                     ssn->data_first_seen_dir = APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER;
                     AppLayerDecoderEventsSetEventRaw(&p->app_layer_events,
                                                      APPLAYER_PROTO_DETECTION_SKIPPED);
+                /* in case of really low TS data (e.g. 4 bytes) we can have
+                 * the PP complete, PM not complete (depth not reached) and
+                 * the TC side also not recognized (proto unknown) */
+                } else if (size_tc > 100000 &&
+                           FLOW_IS_PP_DONE(f, STREAM_TOSERVER) && !(FLOW_IS_PM_DONE(f, STREAM_TOSERVER)) &&
+                           (!FLOW_IS_PM_DONE(f, STREAM_TOCLIENT) && !FLOW_IS_PP_DONE(f, STREAM_TOCLIENT)))
+                {
+                    DisableAppLayer(f);
+                    ssn->data_first_seen_dir = APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER;
+                    AppLayerDecoderEventsSetEventRaw(&p->app_layer_events,
+                                                     APPLAYER_PROTO_DETECTION_SKIPPED);
                 }
-            }
+             }
         }
     } else {
         SCLogDebug("stream data (len %" PRIu32 " alproto "
diff --git a/framework/src/suricata/src/decode.c b/framework/src/suricata/src/decode.c
index 0dd9fa86..4be4b9e7 100644
--- a/framework/src/suricata/src/decode.c
+++ b/framework/src/suricata/src/decode.c
@@ -403,6 +403,7 @@ void DecodeRegisterPerfCounters(DecodeThreadVars *dtv, ThreadVars *tv)
     dtv->counter_avg_pkt_size = StatsRegisterAvgCounter("decoder.avg_pkt_size", tv);
     dtv->counter_max_pkt_size = StatsRegisterMaxCounter("decoder.max_pkt_size", tv);
     dtv->counter_erspan = StatsRegisterMaxCounter("decoder.erspan", tv);
+    dtv->counter_flow_memcap = StatsRegisterCounter("flow.memcap", tv);
 
     dtv->counter_defrag_ipv4_fragments =
         StatsRegisterCounter("defrag.ipv4.fragments", tv);
diff --git a/framework/src/suricata/src/decode.h b/framework/src/suricata/src/decode.h
index 2f322a02..f57dcea9 100644
--- a/framework/src/suricata/src/decode.h
+++ b/framework/src/suricata/src/decode.h
@@ -624,6 +624,8 @@ typedef struct DecodeThreadVars_
     uint16_t counter_defrag_ipv6_timeouts;
     uint16_t counter_defrag_max_hit;
 
+    uint16_t counter_flow_memcap;
+
     /* thread data for flow logging api: only used at forced
      * flow recycle during lookups */
     void *output_flow_thread_data;
diff --git a/framework/src/suricata/src/defrag-hash.c b/framework/src/suricata/src/defrag-hash.c
index 9cb377e5..e37a0873 100644
--- a/framework/src/suricata/src/defrag-hash.c
+++ b/framework/src/suricata/src/defrag-hash.c
@@ -96,24 +96,13 @@ static void DefragTrackerInit(DefragTracker *dt, Packet *p)
     dt->vlan_id[1] = p->vlan_id[1];
     dt->policy = DefragGetOsPolicy(p);
     dt->host_timeout = DefragPolicyGetHostTimeout(p);
+    dt->remove = 0;
+    dt->seen_last = 0;
 
     TAILQ_INIT(&dt->frags);
     (void) DefragTrackerIncrUsecnt(dt);
 }
 
-static DefragTracker *DefragTrackerNew(Packet *p)
-{
-    DefragTracker *dt = DefragTrackerAlloc();
-    if (dt == NULL)
-        goto error;
-
-    DefragTrackerInit(dt, p);
-    return dt;
-
-error:
-    return NULL;
-}
-
 void DefragTrackerRelease(DefragTracker *t)
 {
     (void) DefragTrackerDecrUsecnt(t);
@@ -467,7 +456,7 @@ static DefragTracker *DefragTrackerGetNew(Packet *p)
             /* freed a tracker, but it's unlocked */
         } else {
             /* now see if we can alloc a new tracker */
-            dt = DefragTrackerNew(p);
+            dt = DefragTrackerAlloc();
             if (dt == NULL) {
                 return NULL;
             }
@@ -526,7 +515,7 @@ DefragTracker *DefragGetTrackerFromHash (Packet *p)
     dt = hb->head;
 
     /* see if this is the tracker we are looking for */
-    if (DefragTrackerCompare(dt, p) == 0) {
+    if (dt->remove || DefragTrackerCompare(dt, p) == 0) {
         DefragTracker *pdt = NULL; /* previous tracker */
 
         while (dt) {
diff --git a/framework/src/suricata/src/defrag.c b/framework/src/suricata/src/defrag.c
index 7418b93c..1d86c448 100644
--- a/framework/src/suricata/src/defrag.c
+++ b/framework/src/suricata/src/defrag.c
@@ -337,6 +337,10 @@ Defrag4Reassemble(ThreadVars *tv, DefragTracker *tracker, Packet *p)
             if (frag->offset + frag->data_len > fragmentable_len)
                 fragmentable_len = frag->offset + frag->data_len;
         }
+
+        if (!frag->more_frags) {
+            break;
+        }
     }
 
     SCLogDebug("ip_hdr_offset %u, hlen %u, fragmentable_len %u",
@@ -460,6 +464,10 @@ Defrag6Reassemble(ThreadVars *tv, DefragTracker *tracker, Packet *p)
             if (frag->offset + frag->data_len > fragmentable_len)
                 fragmentable_len = frag->offset + frag->data_len;
         }
+
+        if (!frag->more_frags) {
+            break;
+        }
     }
 
     rp->ip6h = (IPV6Hdr *)(GET_PKT_DATA(rp) + ip_hdr_offset);
@@ -747,6 +755,7 @@ insert:
     new->data_len = data_len - ltrim;
     new->ip_hdr_offset = ip_hdr_offset;
     new->frag_hdr_offset = frag_hdr_offset;
+    new->more_frags = more_frags;
 #ifdef DEBUG
     new->pcap_cnt = pcap_cnt;
 #endif
@@ -2328,6 +2337,198 @@ end:
     return ret;
 }
 
+static int DefragTrackerReuseTest(void)
+{
+    int ret = 0;
+    int id = 1;
+    Packet *p1 = NULL;
+    DefragTracker *tracker1 = NULL, *tracker2 = NULL;
+
+    DefragInit();
+
+    /* Build a packet, its not a fragment but shouldn't matter for
+     * this test. */
+    p1 = BuildTestPacket(id, 0, 0, 'A', 8);
+    if (p1 == NULL) {
+        goto end;
+    }
+
+    /* Get a tracker. It shouldn't look like its already in use. */
+    tracker1 = DefragGetTracker(NULL, NULL, p1);
+    if (tracker1 == NULL) {
+        goto end;
+    }
+    if (tracker1->seen_last) {
+        goto end;
+    }
+    if (tracker1->remove) {
+        goto end;
+    }
+    DefragTrackerRelease(tracker1);
+
+    /* Get a tracker again, it should be the same one. */
+    tracker2 = DefragGetTracker(NULL, NULL, p1);
+    if (tracker2 == NULL) {
+        goto end;
+    }
+    if (tracker2 != tracker1) {
+        goto end;
+    }
+    DefragTrackerRelease(tracker1);
+
+    /* Now mark the tracker for removal. It should not be returned
+     * when we get a tracker for a packet that may have the same
+     * attributes. */
+    tracker1->remove = 1;
+
+    tracker2 = DefragGetTracker(NULL, NULL, p1);
+    if (tracker2 == NULL) {
+        goto end;
+    }
+    if (tracker2 == tracker1) {
+        goto end;
+    }
+    if (tracker2->remove) {
+        goto end;
+    }
+
+    ret = 1;
+end:
+    if (p1 != NULL) {
+        SCFree(p1);
+    }
+    DefragDestroy();
+    return ret;
+}
+
+/**
+ * IPV4: Test the case where you have a packet fragmented in 3 parts
+ * and send like:
+ * - Offset: 2; MF: 1
+ * - Offset: 0; MF: 1
+ * - Offset: 1; MF: 0
+ *
+ * Only the fragments with offset 0 and 1 should be reassembled.
+ */
+static int DefragMfIpv4Test(void)
+{
+    int retval = 0;
+    int ip_id = 9;
+    Packet *p = NULL;
+
+    DefragInit();
+
+    Packet *p1 = BuildTestPacket(ip_id, 2, 1, 'C', 8);
+    Packet *p2 = BuildTestPacket(ip_id, 0, 1, 'A', 8);
+    Packet *p3 = BuildTestPacket(ip_id, 1, 0, 'B', 8);
+    if (p1 == NULL || p2 == NULL || p3 == NULL) {
+        goto end;
+    }
+
+    p = Defrag(NULL, NULL, p1, NULL);
+    if (p != NULL) {
+        goto end;
+    }
+
+    p = Defrag(NULL, NULL, p2, NULL);
+    if (p != NULL) {
+        goto end;
+    }
+
+    /* This should return a packet as MF=0. */
+    p = Defrag(NULL, NULL, p3, NULL);
+    if (p == NULL) {
+        goto end;
+    }
+
+    /* Expected IP length is 20 + 8 + 8 = 36 as only 2 of the
+     * fragments should be in the re-assembled packet. */
+    if (IPV4_GET_IPLEN(p) != 36) {
+        goto end;
+    }
+
+    retval = 1;
+end:
+    if (p1 != NULL) {
+        SCFree(p1);
+    }
+    if (p2 != NULL) {
+        SCFree(p2);
+    }
+    if (p3 != NULL) {
+        SCFree(p3);
+    }
+    if (p != NULL) {
+        SCFree(p);
+    }
+    DefragDestroy();
+    return retval;
+}
+
+/**
+ * IPV6: Test the case where you have a packet fragmented in 3 parts
+ * and send like:
+ * - Offset: 2; MF: 1
+ * - Offset: 0; MF: 1
+ * - Offset: 1; MF: 0
+ *
+ * Only the fragments with offset 0 and 1 should be reassembled.
+ */
+static int DefragMfIpv6Test(void)
+{
+    int retval = 0;
+    int ip_id = 9;
+    Packet *p = NULL;
+
+    DefragInit();
+
+    Packet *p1 = IPV6BuildTestPacket(ip_id, 2, 1, 'C', 8);
+    Packet *p2 = IPV6BuildTestPacket(ip_id, 0, 1, 'A', 8);
+    Packet *p3 = IPV6BuildTestPacket(ip_id, 1, 0, 'B', 8);
+    if (p1 == NULL || p2 == NULL || p3 == NULL) {
+        goto end;
+    }
+
+    p = Defrag(NULL, NULL, p1, NULL);
+    if (p != NULL) {
+        goto end;
+    }
+
+    p = Defrag(NULL, NULL, p2, NULL);
+    if (p != NULL) {
+        goto end;
+    }
+
+    /* This should return a packet as MF=0. */
+    p = Defrag(NULL, NULL, p3, NULL);
+    if (p == NULL) {
+        goto end;
+    }
+
+    /* For IPv6 the expected length is just the length of the payload
+     * of 2 fragments, so 16. */
+    if (IPV6_GET_PLEN(p) != 16) {
+        goto end;
+    }
+
+    retval = 1;
+end:
+    if (p1 != NULL) {
+        SCFree(p1);
+    }
+    if (p2 != NULL) {
+        SCFree(p2);
+    }
+    if (p3 != NULL) {
+        SCFree(p3);
+    }
+    if (p != NULL) {
+        SCFree(p);
+    }
+    DefragDestroy();
+    return retval;
+}
+
 #endif /* UNITTESTS */
 
 void
@@ -2373,9 +2574,11 @@ DefragRegisterTests(void)
 
     UtRegisterTest("DefragVlanTest", DefragVlanTest, 1);
     UtRegisterTest("DefragVlanQinQTest", DefragVlanQinQTest, 1);
-
+    UtRegisterTest("DefragTrackerReuseTest", DefragTrackerReuseTest, 1);
     UtRegisterTest("DefragTimeoutTest",
         DefragTimeoutTest, 1);
+    UtRegisterTest("DefragMfIpv4Test", DefragMfIpv4Test, 1);
+    UtRegisterTest("DefragMfIpv6Test", DefragMfIpv6Test, 1);
 #endif /* UNITTESTS */
 }
 
diff --git a/framework/src/suricata/src/defrag.h b/framework/src/suricata/src/defrag.h
index 8bd0325a..2dfaec60 100644
--- a/framework/src/suricata/src/defrag.h
+++ b/framework/src/suricata/src/defrag.h
@@ -71,20 +71,6 @@ typedef struct Frag_ {
     TAILQ_ENTRY(Frag_) next;    /**< Pointer to next fragment for tailq. */
 } Frag;
 
-/** \brief Reset tracker fields except "lock" */
-#define DEFRAG_TRACKER_RESET(t) { \
-    (t)->timeout = 0; \
-    (t)->id = 0; \
-    (t)->policy = 0; \
-    (t)->af = 0; \
-    (t)->seen_last = 0; \
-    (t)->remove = 0; \
-    CLEAR_ADDR(&(t)->src_addr); \
-    CLEAR_ADDR(&(t)->dst_addr); \
-    (t)->frags.tqh_first = NULL; \
-    (t)->frags.tqh_last = NULL; \
-}
-
 /**
  * A defragmentation tracker.  Used to track fragments that make up a
  * single packet.
diff --git a/framework/src/suricata/src/detect-engine-filedata-smtp.c b/framework/src/suricata/src/detect-engine-filedata-smtp.c
index dc50d8c7..829832f9 100644
--- a/framework/src/suricata/src/detect-engine-filedata-smtp.c
+++ b/framework/src/suricata/src/detect-engine-filedata-smtp.c
@@ -267,7 +267,7 @@ int DetectEngineRunSMTPMpm(DetectEngineCtx *de_ctx,
     uint32_t cnt = 0;
     uint32_t buffer_len = 0;
     uint32_t stream_start_offset = 0;
-    uint8_t *buffer = 0;
+    uint8_t *buffer = NULL;
 
     if (ffc != NULL) {
         File *file = ffc->head;
@@ -278,11 +278,10 @@ int DetectEngineRunSMTPMpm(DetectEngineCtx *de_ctx,
                                                     flags,
                                                     &buffer_len,
                                                     &stream_start_offset);
-
             if (buffer_len == 0)
                 goto end;
 
-            cnt = SMTPFiledataPatternSearch(det_ctx, buffer, buffer_len, flags);
+            cnt += SMTPFiledataPatternSearch(det_ctx, buffer, buffer_len, flags);
         }
     }
 end:
diff --git a/framework/src/suricata/src/detect-engine-mpm.c b/framework/src/suricata/src/detect-engine-mpm.c
index c34ef252..7fd532e5 100644
--- a/framework/src/suricata/src/detect-engine-mpm.c
+++ b/framework/src/suricata/src/detect-engine-mpm.c
@@ -58,6 +58,7 @@
 #ifdef __SC_CUDA_SUPPORT__
 #include "util-mpm-ac.h"
 #endif
+#include "util-validate.h"
 
 /** \todo make it possible to use multiple pattern matcher algorithms next to
           each other. */
@@ -83,7 +84,7 @@ int SignatureHasPacketContent(Signature *s)
         SCReturnInt(0);
     }
 
-    if (!(s->proto.proto[6 / 8] & 1 << (6 % 8))) {
+    if (!(s->proto.proto[IPPROTO_TCP / 8] & 1 << (IPPROTO_TCP % 8))) {
         SCReturnInt(1);
     }
 
@@ -117,7 +118,7 @@ int SignatureHasStreamContent(Signature *s)
         SCReturnInt(0);
     }
 
-    if (!(s->proto.proto[6 / 8] & 1 << (6 % 8))) {
+    if (!(s->proto.proto[IPPROTO_TCP / 8] & 1 << (IPPROTO_TCP % 8))) {
         SCReturnInt(0);
     }
 
@@ -170,7 +171,7 @@ uint16_t PatternMatchDefaultMatcher(void)
  done:
 #ifdef __tile__
     if (mpm_algo_val == MPM_AC)
-      mpm_algo_val = MPM_AC_TILE;
+        mpm_algo_val = MPM_AC_TILE;
 #endif
 
     return mpm_algo_val;
@@ -184,15 +185,13 @@ uint32_t PacketPatternSearchWithStreamCtx(DetectEngineThreadCtx *det_ctx,
     uint32_t ret = 0;
 
     if (p->flowflags & FLOW_PKT_TOSERVER) {
-        if (det_ctx->sgh->mpm_stream_ctx_ts == NULL)
-            SCReturnInt(0);
+        DEBUG_VALIDATE_BUG_ON(det_ctx->sgh->mpm_stream_ctx_ts == NULL);
 
         ret = mpm_table[det_ctx->sgh->mpm_stream_ctx_ts->mpm_type].
             Search(det_ctx->sgh->mpm_stream_ctx_ts, &det_ctx->mtc, &det_ctx->pmq,
                    p->payload, p->payload_len);
     } else {
-        if (det_ctx->sgh->mpm_stream_ctx_tc == NULL)
-            SCReturnInt(0);
+        DEBUG_VALIDATE_BUG_ON(det_ctx->sgh->mpm_stream_ctx_tc == NULL);
 
         ret = mpm_table[det_ctx->sgh->mpm_stream_ctx_tc->mpm_type].
             Search(det_ctx->sgh->mpm_stream_ctx_tc, &det_ctx->mtc, &det_ctx->pmq,
@@ -231,8 +230,7 @@ uint32_t PacketPatternSearch(DetectEngineThreadCtx *det_ctx, Packet *p)
     } else {
         mpm_ctx = det_ctx->sgh->mpm_proto_other_ctx;
     }
-
-    if (mpm_ctx == NULL)
+    if (unlikely(mpm_ctx == NULL))
         SCReturnInt(0);
 
 #ifdef __SC_CUDA_SUPPORT__
@@ -269,16 +267,13 @@ uint32_t UriPatternSearch(DetectEngineThreadCtx *det_ctx,
     SCEnter();
 
     uint32_t ret;
-    if (flags & STREAM_TOSERVER) {
-        if (det_ctx->sgh->mpm_uri_ctx_ts == NULL)
-            SCReturnUInt(0U);
 
-        ret = mpm_table[det_ctx->sgh->mpm_uri_ctx_ts->mpm_type].
-            Search(det_ctx->sgh->mpm_uri_ctx_ts,
-                   &det_ctx->mtcu, &det_ctx->pmq, uri, uri_len);
-    } else {
-        BUG_ON(1);
-    }
+    DEBUG_VALIDATE_BUG_ON(flags & STREAM_TOCLIENT);
+    DEBUG_VALIDATE_BUG_ON(det_ctx->sgh->mpm_uri_ctx_ts == NULL);
+
+    ret = mpm_table[det_ctx->sgh->mpm_uri_ctx_ts->mpm_type].
+        Search(det_ctx->sgh->mpm_uri_ctx_ts,
+                &det_ctx->mtcu, &det_ctx->pmq, uri, uri_len);
 
     //PrintRawDataFp(stdout, uri, uri_len);
 
@@ -300,16 +295,13 @@ uint32_t HttpClientBodyPatternSearch(DetectEngineThreadCtx *det_ctx,
     SCEnter();
 
     uint32_t ret;
-    if (flags & STREAM_TOSERVER) {
-        if (det_ctx->sgh->mpm_hcbd_ctx_ts == NULL)
-            SCReturnUInt(0);
 
-        ret = mpm_table[det_ctx->sgh->mpm_hcbd_ctx_ts->mpm_type].
-            Search(det_ctx->sgh->mpm_hcbd_ctx_ts, &det_ctx->mtcu,
-                   &det_ctx->pmq, body, body_len);
-    } else {
-        BUG_ON(1);
-    }
+    DEBUG_VALIDATE_BUG_ON(flags & STREAM_TOCLIENT);
+    DEBUG_VALIDATE_BUG_ON(det_ctx->sgh->mpm_hcbd_ctx_ts == NULL);
+
+    ret = mpm_table[det_ctx->sgh->mpm_hcbd_ctx_ts->mpm_type].
+        Search(det_ctx->sgh->mpm_hcbd_ctx_ts, &det_ctx->mtcu,
+                &det_ctx->pmq, body, body_len);
 
     SCReturnUInt(ret);
 }
@@ -329,16 +321,13 @@ uint32_t HttpServerBodyPatternSearch(DetectEngineThreadCtx *det_ctx,
     SCEnter();
 
     uint32_t ret;
-    if (flags & STREAM_TOSERVER) {
-        BUG_ON(1);
-    } else {
-        if (det_ctx->sgh->mpm_hsbd_ctx_tc == NULL)
-            SCReturnUInt(0);
 
-        ret = mpm_table[det_ctx->sgh->mpm_hsbd_ctx_tc->mpm_type].
-            Search(det_ctx->sgh->mpm_hsbd_ctx_tc, &det_ctx->mtcu,
-                   &det_ctx->pmq, body, body_len);
-    }
+    DEBUG_VALIDATE_BUG_ON(!(flags & STREAM_TOCLIENT));
+    DEBUG_VALIDATE_BUG_ON(det_ctx->sgh->mpm_hsbd_ctx_tc == NULL);
+
+    ret = mpm_table[det_ctx->sgh->mpm_hsbd_ctx_tc->mpm_type].
+        Search(det_ctx->sgh->mpm_hsbd_ctx_tc, &det_ctx->mtcu,
+                &det_ctx->pmq, body, body_len);
 
     SCReturnUInt(ret);
 }
@@ -359,15 +348,13 @@ uint32_t HttpHeaderPatternSearch(DetectEngineThreadCtx *det_ctx,
 
     uint32_t ret;
     if (flags & STREAM_TOSERVER) {
-        if (det_ctx->sgh->mpm_hhd_ctx_ts == NULL)
-            SCReturnUInt(0);
+        DEBUG_VALIDATE_BUG_ON(det_ctx->sgh->mpm_hhd_ctx_ts == NULL);
 
         ret = mpm_table[det_ctx->sgh->mpm_hhd_ctx_ts->mpm_type].
             Search(det_ctx->sgh->mpm_hhd_ctx_ts, &det_ctx->mtcu,
                    &det_ctx->pmq, headers, headers_len);
     } else {
-        if (det_ctx->sgh->mpm_hhd_ctx_tc == NULL)
-            SCReturnUInt(0);
+        DEBUG_VALIDATE_BUG_ON(det_ctx->sgh->mpm_hhd_ctx_tc == NULL);
 
         ret = mpm_table[det_ctx->sgh->mpm_hhd_ctx_tc->mpm_type].
             Search(det_ctx->sgh->mpm_hhd_ctx_tc, &det_ctx->mtcu,
@@ -393,15 +380,13 @@ uint32_t HttpRawHeaderPatternSearch(DetectEngineThreadCtx *det_ctx,
 
     uint32_t ret;
     if (flags & STREAM_TOSERVER) {
-        if (det_ctx->sgh->mpm_hrhd_ctx_ts == NULL)
-            SCReturnUInt(0);
+        DEBUG_VALIDATE_BUG_ON(det_ctx->sgh->mpm_hrhd_ctx_ts == NULL);
 
         ret = mpm_table[det_ctx->sgh->mpm_hrhd_ctx_ts->mpm_type].
             Search(det_ctx->sgh->mpm_hrhd_ctx_ts, &det_ctx->mtcu,
                    &det_ctx->pmq, raw_headers, raw_headers_len);
     } else {
-        if (det_ctx->sgh->mpm_hrhd_ctx_tc == NULL)
-            SCReturnUInt(0);
+        DEBUG_VALIDATE_BUG_ON(det_ctx->sgh->mpm_hrhd_ctx_tc == NULL);
 
         ret = mpm_table[det_ctx->sgh->mpm_hrhd_ctx_tc->mpm_type].
             Search(det_ctx->sgh->mpm_hrhd_ctx_tc, &det_ctx->mtcu,
@@ -426,16 +411,13 @@ uint32_t HttpMethodPatternSearch(DetectEngineThreadCtx *det_ctx,
     SCEnter();
 
     uint32_t ret;
-    if (flags & STREAM_TOSERVER) {
-        if (det_ctx->sgh->mpm_hmd_ctx_ts == NULL)
-            SCReturnUInt(0);
 
-        ret = mpm_table[det_ctx->sgh->mpm_hmd_ctx_ts->mpm_type].
-            Search(det_ctx->sgh->mpm_hmd_ctx_ts, &det_ctx->mtcu,
-                   &det_ctx->pmq, raw_method, raw_method_len);
-    } else {
-        BUG_ON(1);
-    }
+    DEBUG_VALIDATE_BUG_ON(flags & STREAM_TOCLIENT);
+    DEBUG_VALIDATE_BUG_ON(det_ctx->sgh->mpm_hmd_ctx_ts == NULL);
+
+    ret = mpm_table[det_ctx->sgh->mpm_hmd_ctx_ts->mpm_type].
+        Search(det_ctx->sgh->mpm_hmd_ctx_ts, &det_ctx->mtcu,
+                &det_ctx->pmq, raw_method, raw_method_len);
 
     SCReturnUInt(ret);
 }
@@ -456,15 +438,13 @@ uint32_t HttpCookiePatternSearch(DetectEngineThreadCtx *det_ctx,
 
     uint32_t ret;
     if (flags & STREAM_TOSERVER) {
-        if (det_ctx->sgh->mpm_hcd_ctx_ts == NULL)
-            SCReturnUInt(0);
+        DEBUG_VALIDATE_BUG_ON(det_ctx->sgh->mpm_hcd_ctx_ts == NULL);
 
         ret = mpm_table[det_ctx->sgh->mpm_hcd_ctx_ts->mpm_type].
             Search(det_ctx->sgh->mpm_hcd_ctx_ts, &det_ctx->mtcu,
                    &det_ctx->pmq, cookie, cookie_len);
     } else {
-        if (det_ctx->sgh->mpm_hcd_ctx_tc == NULL)
-            SCReturnUInt(0);
+        DEBUG_VALIDATE_BUG_ON(det_ctx->sgh->mpm_hcd_ctx_tc == NULL);
 
         ret = mpm_table[det_ctx->sgh->mpm_hcd_ctx_tc->mpm_type].
             Search(det_ctx->sgh->mpm_hcd_ctx_tc, &det_ctx->mtcu,
@@ -489,16 +469,13 @@ uint32_t HttpRawUriPatternSearch(DetectEngineThreadCtx *det_ctx,
     SCEnter();
 
     uint32_t ret;
-    if (flags & STREAM_TOSERVER) {
-        if (det_ctx->sgh->mpm_hrud_ctx_ts == NULL)
-            SCReturnUInt(0);
 
-        ret = mpm_table[det_ctx->sgh->mpm_hrud_ctx_ts->mpm_type].
-            Search(det_ctx->sgh->mpm_hrud_ctx_ts, &det_ctx->mtcu,
-                   &det_ctx->pmq, uri, uri_len);
-    } else {
-        BUG_ON(1);
-    }
+    DEBUG_VALIDATE_BUG_ON(flags & STREAM_TOCLIENT);
+    DEBUG_VALIDATE_BUG_ON(det_ctx->sgh->mpm_hrud_ctx_ts == NULL);
+
+    ret = mpm_table[det_ctx->sgh->mpm_hrud_ctx_ts->mpm_type].
+        Search(det_ctx->sgh->mpm_hrud_ctx_ts, &det_ctx->mtcu,
+                &det_ctx->pmq, uri, uri_len);
 
     SCReturnUInt(ret);
 }
@@ -518,16 +495,13 @@ uint32_t HttpStatMsgPatternSearch(DetectEngineThreadCtx *det_ctx,
     SCEnter();
 
     uint32_t ret;
-    if (flags & STREAM_TOSERVER) {
-        BUG_ON(1);
-    } else {
-        if (det_ctx->sgh->mpm_hsmd_ctx_tc == NULL)
-            SCReturnUInt(0);
 
-        ret = mpm_table[det_ctx->sgh->mpm_hsmd_ctx_tc->mpm_type].
-            Search(det_ctx->sgh->mpm_hsmd_ctx_tc, &det_ctx->mtcu,
-                   &det_ctx->pmq, stat_msg, stat_msg_len);
-    }
+    DEBUG_VALIDATE_BUG_ON(!(flags & STREAM_TOCLIENT));
+    DEBUG_VALIDATE_BUG_ON(det_ctx->sgh->mpm_hsmd_ctx_tc == NULL);
+
+    ret = mpm_table[det_ctx->sgh->mpm_hsmd_ctx_tc->mpm_type].
+        Search(det_ctx->sgh->mpm_hsmd_ctx_tc, &det_ctx->mtcu,
+                &det_ctx->pmq, stat_msg, stat_msg_len);
 
     SCReturnUInt(ret);
 }
@@ -547,16 +521,13 @@ uint32_t HttpStatCodePatternSearch(DetectEngineThreadCtx *det_ctx,
     SCEnter();
 
     uint32_t ret;
-    if (flags & STREAM_TOSERVER) {
-        BUG_ON(1);
-    } else {
-        if (det_ctx->sgh->mpm_hscd_ctx_tc == NULL)
-            SCReturnUInt(0);
 
-        ret = mpm_table[det_ctx->sgh->mpm_hscd_ctx_tc->mpm_type].
-            Search(det_ctx->sgh->mpm_hscd_ctx_tc, &det_ctx->mtcu,
-                   &det_ctx->pmq, stat_code, stat_code_len);
-    }
+    DEBUG_VALIDATE_BUG_ON(!(flags & STREAM_TOCLIENT));
+    DEBUG_VALIDATE_BUG_ON(det_ctx->sgh->mpm_hscd_ctx_tc == NULL);
+
+    ret = mpm_table[det_ctx->sgh->mpm_hscd_ctx_tc->mpm_type].
+        Search(det_ctx->sgh->mpm_hscd_ctx_tc, &det_ctx->mtcu,
+                &det_ctx->pmq, stat_code, stat_code_len);
 
     SCReturnUInt(ret);
 }
@@ -576,16 +547,13 @@ uint32_t HttpUAPatternSearch(DetectEngineThreadCtx *det_ctx,
     SCEnter();
 
     uint32_t ret;
-    if (flags & STREAM_TOSERVER) {
-        if (det_ctx->sgh->mpm_huad_ctx_ts == NULL)
-            SCReturnUInt(0);
 
-        ret = mpm_table[det_ctx->sgh->mpm_huad_ctx_ts->mpm_type].
-            Search(det_ctx->sgh->mpm_huad_ctx_ts, &det_ctx->mtcu,
-                   &det_ctx->pmq, ua, ua_len);
-    } else {
-        BUG_ON(1);
-    }
+    DEBUG_VALIDATE_BUG_ON(flags & STREAM_TOCLIENT);
+    DEBUG_VALIDATE_BUG_ON(det_ctx->sgh->mpm_huad_ctx_ts == NULL);
+
+    ret = mpm_table[det_ctx->sgh->mpm_huad_ctx_ts->mpm_type].
+        Search(det_ctx->sgh->mpm_huad_ctx_ts, &det_ctx->mtcu,
+                &det_ctx->pmq, ua, ua_len);
 
     SCReturnUInt(ret);
 }
@@ -606,16 +574,13 @@ uint32_t HttpHHPatternSearch(DetectEngineThreadCtx *det_ctx,
     SCEnter();
 
     uint32_t ret;
-    if (flags & STREAM_TOSERVER) {
-        if (det_ctx->sgh->mpm_hhhd_ctx_ts == NULL)
-            SCReturnUInt(0);
 
-        ret = mpm_table[det_ctx->sgh->mpm_hhhd_ctx_ts->mpm_type].
-            Search(det_ctx->sgh->mpm_hhhd_ctx_ts, &det_ctx->mtcu,
-                   &det_ctx->pmq, hh, hh_len);
-    } else {
-        BUG_ON(1);
-    }
+    DEBUG_VALIDATE_BUG_ON(flags & STREAM_TOCLIENT);
+    DEBUG_VALIDATE_BUG_ON(det_ctx->sgh->mpm_hhhd_ctx_ts == NULL);
+
+    ret = mpm_table[det_ctx->sgh->mpm_hhhd_ctx_ts->mpm_type].
+        Search(det_ctx->sgh->mpm_hhhd_ctx_ts, &det_ctx->mtcu,
+                &det_ctx->pmq, hh, hh_len);
 
     SCReturnUInt(ret);
 }
@@ -636,16 +601,13 @@ uint32_t HttpHRHPatternSearch(DetectEngineThreadCtx *det_ctx,
     SCEnter();
 
     uint32_t ret;
-    if (flags & STREAM_TOSERVER) {
-        if (det_ctx->sgh->mpm_hrhhd_ctx_ts == NULL)
-            SCReturnUInt(0);
 
-        ret = mpm_table[det_ctx->sgh->mpm_hrhhd_ctx_ts->mpm_type].
-            Search(det_ctx->sgh->mpm_hrhhd_ctx_ts, &det_ctx->mtcu,
-                   &det_ctx->pmq, hrh, hrh_len);
-    } else {
-        BUG_ON(1);
-    }
+    DEBUG_VALIDATE_BUG_ON(flags & STREAM_TOCLIENT);
+    DEBUG_VALIDATE_BUG_ON(det_ctx->sgh->mpm_hrhhd_ctx_ts == NULL);
+
+    ret = mpm_table[det_ctx->sgh->mpm_hrhhd_ctx_ts->mpm_type].
+        Search(det_ctx->sgh->mpm_hrhhd_ctx_ts, &det_ctx->mtcu,
+                &det_ctx->pmq, hrh, hrh_len);
 
     SCReturnUInt(ret);
 }
@@ -668,14 +630,12 @@ uint32_t DnsQueryPatternSearch(DetectEngineThreadCtx *det_ctx,
 
     uint32_t ret = 0;
 
-    if (flags & STREAM_TOSERVER) {
-        if (det_ctx->sgh->mpm_dnsquery_ctx_ts == NULL)
-            SCReturnUInt(0);
+    DEBUG_VALIDATE_BUG_ON(flags & STREAM_TOCLIENT);
+    DEBUG_VALIDATE_BUG_ON(det_ctx->sgh->mpm_dnsquery_ctx_ts == NULL);
 
-        ret = mpm_table[det_ctx->sgh->mpm_dnsquery_ctx_ts->mpm_type].
-            Search(det_ctx->sgh->mpm_dnsquery_ctx_ts, &det_ctx->mtcu,
-                   &det_ctx->pmq, buffer, buffer_len);
-    }
+    ret = mpm_table[det_ctx->sgh->mpm_dnsquery_ctx_ts->mpm_type].
+        Search(det_ctx->sgh->mpm_dnsquery_ctx_ts, &det_ctx->mtcu,
+                &det_ctx->pmq, buffer, buffer_len);
 
     SCReturnUInt(ret);
 }
@@ -755,14 +715,12 @@ uint32_t SMTPFiledataPatternSearch(DetectEngineThreadCtx *det_ctx,
 
     uint32_t ret = 0;
 
-    if (flags & STREAM_TOSERVER) {
-        if (det_ctx->sgh->mpm_smtp_filedata_ctx_ts == NULL)
-            SCReturnUInt(0);
+    DEBUG_VALIDATE_BUG_ON(flags & STREAM_TOCLIENT);
+    DEBUG_VALIDATE_BUG_ON(det_ctx->sgh->mpm_smtp_filedata_ctx_ts == NULL);
 
-        ret = mpm_table[det_ctx->sgh->mpm_smtp_filedata_ctx_ts->mpm_type].
-            Search(det_ctx->sgh->mpm_smtp_filedata_ctx_ts, &det_ctx->mtcu,
-                   &det_ctx->pmq, buffer, buffer_len);
-    }
+    ret = mpm_table[det_ctx->sgh->mpm_smtp_filedata_ctx_ts->mpm_type].
+        Search(det_ctx->sgh->mpm_smtp_filedata_ctx_ts, &det_ctx->mtcu,
+                &det_ctx->pmq, buffer, buffer_len);
 
     SCReturnUInt(ret);
 }
@@ -862,10 +820,11 @@ void PatternMatchDestroyGroup(SigGroupHead *sh)
 {
     /* content */
     if (!(sh->flags & SIG_GROUP_HEAD_MPM_COPY)) {
-        SCLogDebug("destroying mpm_ctx %p (sh %p)",
-                   sh->mpm_proto_tcp_ctx_ts, sh);
         if (sh->mpm_proto_tcp_ctx_ts != NULL &&
-            !sh->mpm_proto_tcp_ctx_ts->global) {
+            !sh->mpm_proto_tcp_ctx_ts->global)
+        {
+            SCLogDebug("destroying mpm_ctx %p (sh %p)",
+                    sh->mpm_proto_tcp_ctx_ts, sh);
             mpm_table[sh->mpm_proto_tcp_ctx_ts->mpm_type].
                 DestroyCtx(sh->mpm_proto_tcp_ctx_ts);
             SCFree(sh->mpm_proto_tcp_ctx_ts);
@@ -873,10 +832,11 @@ void PatternMatchDestroyGroup(SigGroupHead *sh)
         /* ready for reuse */
         sh->mpm_proto_tcp_ctx_ts = NULL;
 
-        SCLogDebug("destroying mpm_ctx %p (sh %p)",
-                   sh->mpm_proto_tcp_ctx_tc, sh);
         if (sh->mpm_proto_tcp_ctx_tc != NULL &&
-            !sh->mpm_proto_tcp_ctx_tc->global) {
+            !sh->mpm_proto_tcp_ctx_tc->global)
+        {
+            SCLogDebug("destroying mpm_ctx %p (sh %p)",
+                    sh->mpm_proto_tcp_ctx_tc, sh);
             mpm_table[sh->mpm_proto_tcp_ctx_tc->mpm_type].
                 DestroyCtx(sh->mpm_proto_tcp_ctx_tc);
             SCFree(sh->mpm_proto_tcp_ctx_tc);
@@ -884,10 +844,11 @@ void PatternMatchDestroyGroup(SigGroupHead *sh)
         /* ready for reuse */
         sh->mpm_proto_tcp_ctx_tc = NULL;
 
-        SCLogDebug("destroying mpm_ctx %p (sh %p)",
-                   sh->mpm_proto_udp_ctx_ts, sh);
         if (sh->mpm_proto_udp_ctx_ts != NULL &&
-            !sh->mpm_proto_udp_ctx_ts->global) {
+            !sh->mpm_proto_udp_ctx_ts->global)
+        {
+            SCLogDebug("destroying mpm_ctx %p (sh %p)",
+                    sh->mpm_proto_udp_ctx_ts, sh);
             mpm_table[sh->mpm_proto_udp_ctx_ts->mpm_type].
                 DestroyCtx(sh->mpm_proto_udp_ctx_ts);
             SCFree(sh->mpm_proto_udp_ctx_ts);
@@ -895,10 +856,11 @@ void PatternMatchDestroyGroup(SigGroupHead *sh)
         /* ready for reuse */
         sh->mpm_proto_udp_ctx_ts = NULL;
 
-        SCLogDebug("destroying mpm_ctx %p (sh %p)",
-                   sh->mpm_proto_udp_ctx_tc, sh);
         if (sh->mpm_proto_udp_ctx_tc != NULL &&
-            !sh->mpm_proto_udp_ctx_tc->global) {
+                !sh->mpm_proto_udp_ctx_tc->global)
+        {
+            SCLogDebug("destroying mpm_ctx %p (sh %p)",
+                    sh->mpm_proto_udp_ctx_tc, sh);
             mpm_table[sh->mpm_proto_udp_ctx_tc->mpm_type].
                 DestroyCtx(sh->mpm_proto_udp_ctx_tc);
             SCFree(sh->mpm_proto_udp_ctx_tc);
@@ -906,10 +868,11 @@ void PatternMatchDestroyGroup(SigGroupHead *sh)
         /* ready for reuse */
         sh->mpm_proto_udp_ctx_tc = NULL;
 
-        SCLogDebug("destroying mpm_ctx %p (sh %p)",
-                   sh->mpm_proto_other_ctx, sh);
         if (sh->mpm_proto_other_ctx != NULL &&
-            !sh->mpm_proto_other_ctx->global) {
+                !sh->mpm_proto_other_ctx->global)
+        {
+            SCLogDebug("destroying mpm_ctx %p (sh %p)",
+                    sh->mpm_proto_other_ctx, sh);
             mpm_table[sh->mpm_proto_other_ctx->mpm_type].
                 DestroyCtx(sh->mpm_proto_other_ctx);
             SCFree(sh->mpm_proto_other_ctx);
@@ -921,8 +884,8 @@ void PatternMatchDestroyGroup(SigGroupHead *sh)
     /* uricontent */
     if ((sh->mpm_uri_ctx_ts != NULL) && !(sh->flags & SIG_GROUP_HEAD_MPM_URI_COPY)) {
         if (sh->mpm_uri_ctx_ts != NULL) {
-            SCLogDebug("destroying mpm_uri_ctx %p (sh %p)", sh->mpm_uri_ctx_ts, sh);
             if (!sh->mpm_uri_ctx_ts->global) {
+                SCLogDebug("destroying mpm_uri_ctx %p (sh %p)", sh->mpm_uri_ctx_ts, sh);
                 mpm_table[sh->mpm_uri_ctx_ts->mpm_type].DestroyCtx(sh->mpm_uri_ctx_ts);
                 SCFree(sh->mpm_uri_ctx_ts);
             }
@@ -935,8 +898,8 @@ void PatternMatchDestroyGroup(SigGroupHead *sh)
     if ((sh->mpm_stream_ctx_ts != NULL || sh->mpm_stream_ctx_tc != NULL) &&
         !(sh->flags & SIG_GROUP_HEAD_MPM_STREAM_COPY)) {
         if (sh->mpm_stream_ctx_ts != NULL) {
-            SCLogDebug("destroying mpm_stream_ctx %p (sh %p)", sh->mpm_stream_ctx_ts, sh);
             if (!sh->mpm_stream_ctx_ts->global) {
+                SCLogDebug("destroying mpm_stream_ctx %p (sh %p)", sh->mpm_stream_ctx_ts, sh);
                 mpm_table[sh->mpm_stream_ctx_ts->mpm_type].DestroyCtx(sh->mpm_stream_ctx_ts);
                 SCFree(sh->mpm_stream_ctx_ts);
             }
@@ -944,8 +907,8 @@ void PatternMatchDestroyGroup(SigGroupHead *sh)
             sh->mpm_stream_ctx_ts = NULL;
         }
         if (sh->mpm_stream_ctx_tc != NULL) {
-            SCLogDebug("destroying mpm_stream_ctx %p (sh %p)", sh->mpm_stream_ctx_tc, sh);
             if (!sh->mpm_stream_ctx_tc->global) {
+                SCLogDebug("destroying mpm_stream_ctx %p (sh %p)", sh->mpm_stream_ctx_tc, sh);
                 mpm_table[sh->mpm_stream_ctx_tc->mpm_type].DestroyCtx(sh->mpm_stream_ctx_tc);
                 SCFree(sh->mpm_stream_ctx_tc);
             }
@@ -2380,315 +2343,312 @@ int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh)
         has_co_hrhhd ||
         has_co_dnsquery)
     {
-
         PatternMatchPreparePopulateMpm(de_ctx, sh);
 
-        //if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
-         if (sh->mpm_proto_tcp_ctx_ts != NULL) {
-             if (sh->mpm_proto_tcp_ctx_ts->pattern_cnt == 0) {
-                 MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_proto_tcp_ctx_ts);
-                 sh->mpm_proto_tcp_ctx_ts = NULL;
-             } else {
-                 if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
-                     if (mpm_table[sh->mpm_proto_tcp_ctx_ts->mpm_type].Prepare != NULL) {
-                         mpm_table[sh->mpm_proto_tcp_ctx_ts->mpm_type].
-                             Prepare(sh->mpm_proto_tcp_ctx_ts);
-                     }
-                 }
-             }
-         }
-         if (sh->mpm_proto_tcp_ctx_tc != NULL) {
-             if (sh->mpm_proto_tcp_ctx_tc->pattern_cnt == 0) {
-                 MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_proto_tcp_ctx_tc);
-                 sh->mpm_proto_tcp_ctx_tc = NULL;
-             } else {
-                 if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
-                     if (mpm_table[sh->mpm_proto_tcp_ctx_tc->mpm_type].Prepare != NULL) {
-                         mpm_table[sh->mpm_proto_tcp_ctx_tc->mpm_type].
-                             Prepare(sh->mpm_proto_tcp_ctx_tc);
-                     }
-                 }
-             }
-         }
-
-         if (sh->mpm_proto_udp_ctx_ts != NULL) {
-             if (sh->mpm_proto_udp_ctx_ts->pattern_cnt == 0) {
-                 MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_proto_udp_ctx_ts);
-                 sh->mpm_proto_udp_ctx_ts = NULL;
-             } else {
-                 if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
-                     if (mpm_table[sh->mpm_proto_udp_ctx_ts->mpm_type].Prepare != NULL) {
-                         mpm_table[sh->mpm_proto_udp_ctx_ts->mpm_type].
-                             Prepare(sh->mpm_proto_udp_ctx_ts);
-                     }
-                 }
-             }
-         }
-         if (sh->mpm_proto_udp_ctx_tc != NULL) {
-             if (sh->mpm_proto_udp_ctx_tc->pattern_cnt == 0) {
-                 MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_proto_udp_ctx_tc);
-                 sh->mpm_proto_udp_ctx_tc = NULL;
-             } else {
-                 if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
-                     if (mpm_table[sh->mpm_proto_udp_ctx_tc->mpm_type].Prepare != NULL) {
-                         mpm_table[sh->mpm_proto_udp_ctx_tc->mpm_type].
-                             Prepare(sh->mpm_proto_udp_ctx_tc);
-                     }
-                 }
-             }
-         }
-
-         if (sh->mpm_proto_other_ctx != NULL) {
-             if (sh->mpm_proto_other_ctx->pattern_cnt == 0) {
-                 MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_proto_other_ctx);
-                 sh->mpm_proto_other_ctx = NULL;
-             } else {
-                 if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
-                     if (mpm_table[sh->mpm_proto_other_ctx->mpm_type].Prepare != NULL) {
-                         mpm_table[sh->mpm_proto_other_ctx->mpm_type].
-                             Prepare(sh->mpm_proto_other_ctx);
-                     }
-                 }
-             }
-         }
-
-         if (sh->mpm_stream_ctx_ts != NULL) {
-             if (sh->mpm_stream_ctx_ts->pattern_cnt == 0) {
-                 MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_stream_ctx_ts);
-                 sh->mpm_stream_ctx_ts = NULL;
-             } else {
-                 if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
-                     if (mpm_table[sh->mpm_stream_ctx_ts->mpm_type].Prepare != NULL)
-                         mpm_table[sh->mpm_stream_ctx_ts->mpm_type].Prepare(sh->mpm_stream_ctx_ts);
-                 }
-             }
-         }
-         if (sh->mpm_stream_ctx_tc != NULL) {
-             if (sh->mpm_stream_ctx_tc->pattern_cnt == 0) {
-                 MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_stream_ctx_tc);
-                 sh->mpm_stream_ctx_tc = NULL;
-             } else {
-                 if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
-                     if (mpm_table[sh->mpm_stream_ctx_tc->mpm_type].Prepare != NULL)
-                         mpm_table[sh->mpm_stream_ctx_tc->mpm_type].Prepare(sh->mpm_stream_ctx_tc);
-                 }
-             }
-         }
-
-         if (sh->mpm_uri_ctx_ts != NULL) {
-             if (sh->mpm_uri_ctx_ts->pattern_cnt == 0) {
-                 MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_uri_ctx_ts);
-                 sh->mpm_uri_ctx_ts = NULL;
-             } else {
-                 if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
-                     if (mpm_table[sh->mpm_uri_ctx_ts->mpm_type].Prepare != NULL)
-                         mpm_table[sh->mpm_uri_ctx_ts->mpm_type].Prepare(sh->mpm_uri_ctx_ts);
-                 }
-             }
-         }
-
-         if (sh->mpm_hcbd_ctx_ts != NULL) {
-             if (sh->mpm_hcbd_ctx_ts->pattern_cnt == 0) {
-                 MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hcbd_ctx_ts);
-                 sh->mpm_hcbd_ctx_ts = NULL;
-             } else {
-                 if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
-                     if (mpm_table[sh->mpm_hcbd_ctx_ts->mpm_type].Prepare != NULL)
-                         mpm_table[sh->mpm_hcbd_ctx_ts->mpm_type].Prepare(sh->mpm_hcbd_ctx_ts);
-                 }
-             }
-         }
-
-         if (sh->mpm_hsbd_ctx_tc != NULL) {
-             if (sh->mpm_hsbd_ctx_tc->pattern_cnt == 0) {
-                 MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hsbd_ctx_tc);
-                 sh->mpm_hsbd_ctx_tc = NULL;
-             } else {
-                 if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
-                     if (mpm_table[sh->mpm_hsbd_ctx_tc->mpm_type].Prepare != NULL)
-                         mpm_table[sh->mpm_hsbd_ctx_tc->mpm_type].Prepare(sh->mpm_hsbd_ctx_tc);
-                 }
-             }
-         }
-
-         if (sh->mpm_smtp_filedata_ctx_ts != NULL) {
-             if (sh->mpm_smtp_filedata_ctx_ts->pattern_cnt == 0) {
-                 MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_smtp_filedata_ctx_ts);
-                 sh->mpm_smtp_filedata_ctx_ts = NULL;
-             } else {
-                 if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
-                     if (mpm_table[sh->mpm_smtp_filedata_ctx_ts->mpm_type].Prepare != NULL) {
-                         mpm_table[sh->mpm_smtp_filedata_ctx_ts->mpm_type].Prepare(sh->mpm_smtp_filedata_ctx_ts);
-                     }
-                 }
-             }
-         }
-
-         if (sh->mpm_hhd_ctx_ts != NULL) {
-             if (sh->mpm_hhd_ctx_ts->pattern_cnt == 0) {
-                 MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hhd_ctx_ts);
-                 sh->mpm_hhd_ctx_ts = NULL;
-             } else {
-                 if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
-                     if (mpm_table[sh->mpm_hhd_ctx_ts->mpm_type].Prepare != NULL)
-                         mpm_table[sh->mpm_hhd_ctx_ts->mpm_type].Prepare(sh->mpm_hhd_ctx_ts);
-                 }
-             }
-         }
-         if (sh->mpm_hhd_ctx_tc != NULL) {
-             if (sh->mpm_hhd_ctx_tc->pattern_cnt == 0) {
-                 MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hhd_ctx_tc);
-                 sh->mpm_hhd_ctx_tc = NULL;
-             } else {
-                 if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
-                     if (mpm_table[sh->mpm_hhd_ctx_tc->mpm_type].Prepare != NULL)
-                         mpm_table[sh->mpm_hhd_ctx_tc->mpm_type].Prepare(sh->mpm_hhd_ctx_tc);
-                 }
-             }
-         }
-
-         if (sh->mpm_hrhd_ctx_ts != NULL) {
-             if (sh->mpm_hrhd_ctx_ts->pattern_cnt == 0) {
-                 MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hrhd_ctx_ts);
-                 sh->mpm_hrhd_ctx_ts = NULL;
-             } else {
-                 if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
-                     if (mpm_table[sh->mpm_hrhd_ctx_ts->mpm_type].Prepare != NULL)
-                         mpm_table[sh->mpm_hrhd_ctx_ts->mpm_type].Prepare(sh->mpm_hrhd_ctx_ts);
-                 }
-             }
-         }
-         if (sh->mpm_hrhd_ctx_tc != NULL) {
-             if (sh->mpm_hrhd_ctx_tc->pattern_cnt == 0) {
-                 MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hrhd_ctx_tc);
-                 sh->mpm_hrhd_ctx_tc = NULL;
-             } else {
-                 if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
-                     if (mpm_table[sh->mpm_hrhd_ctx_tc->mpm_type].Prepare != NULL)
-                         mpm_table[sh->mpm_hrhd_ctx_tc->mpm_type].Prepare(sh->mpm_hrhd_ctx_tc);
-                 }
-             }
-         }
-
-         if (sh->mpm_hmd_ctx_ts != NULL) {
-             if (sh->mpm_hmd_ctx_ts->pattern_cnt == 0) {
-                 MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hmd_ctx_ts);
-                 sh->mpm_hmd_ctx_ts = NULL;
-             } else {
-                 if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
-                     if (mpm_table[sh->mpm_hmd_ctx_ts->mpm_type].Prepare != NULL)
-                         mpm_table[sh->mpm_hmd_ctx_ts->mpm_type].Prepare(sh->mpm_hmd_ctx_ts);
-                 }
-             }
-         }
-
-         if (sh->mpm_hcd_ctx_ts != NULL) {
-             if (sh->mpm_hcd_ctx_ts->pattern_cnt == 0) {
-                 MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hcd_ctx_ts);
-                 sh->mpm_hcd_ctx_ts = NULL;
-             } else {
-                 if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
-                     if (mpm_table[sh->mpm_hcd_ctx_ts->mpm_type].Prepare != NULL)
-                         mpm_table[sh->mpm_hcd_ctx_ts->mpm_type].Prepare(sh->mpm_hcd_ctx_ts);
-                 }
-             }
-         }
-         if (sh->mpm_hcd_ctx_tc != NULL) {
-             if (sh->mpm_hcd_ctx_tc->pattern_cnt == 0) {
-                 MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hcd_ctx_tc);
-                 sh->mpm_hcd_ctx_tc = NULL;
-             } else {
-                 if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
-                     if (mpm_table[sh->mpm_hcd_ctx_tc->mpm_type].Prepare != NULL)
-                         mpm_table[sh->mpm_hcd_ctx_tc->mpm_type].Prepare(sh->mpm_hcd_ctx_tc);
-                 }
-             }
-         }
-
-         if (sh->mpm_hrud_ctx_ts != NULL) {
-             if (sh->mpm_hrud_ctx_ts->pattern_cnt == 0) {
-                 MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hrud_ctx_ts);
-                 sh->mpm_hrud_ctx_ts = NULL;
-             } else {
-                 if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
-                     if (mpm_table[sh->mpm_hrud_ctx_ts->mpm_type].Prepare != NULL)
-                         mpm_table[sh->mpm_hrud_ctx_ts->mpm_type].Prepare(sh->mpm_hrud_ctx_ts);
-                 }
-             }
-         }
-
-         if (sh->mpm_hsmd_ctx_tc != NULL) {
-             if (sh->mpm_hsmd_ctx_tc->pattern_cnt == 0) {
-                 MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hsmd_ctx_tc);
-                 sh->mpm_hsmd_ctx_tc = NULL;
-             } else {
-                 if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
-                     if (mpm_table[sh->mpm_hsmd_ctx_tc->mpm_type].Prepare != NULL)
-                         mpm_table[sh->mpm_hsmd_ctx_tc->mpm_type].Prepare(sh->mpm_hsmd_ctx_tc);
-                 }
-             }
-         }
-
-         if (sh->mpm_hscd_ctx_tc != NULL) {
-             if (sh->mpm_hscd_ctx_tc->pattern_cnt == 0) {
-                 MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hscd_ctx_tc);
-                 sh->mpm_hscd_ctx_tc = NULL;
-             } else {
-                 if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
-                     if (mpm_table[sh->mpm_hscd_ctx_tc->mpm_type].Prepare != NULL)
-                         mpm_table[sh->mpm_hscd_ctx_tc->mpm_type].Prepare(sh->mpm_hscd_ctx_tc);
-                 }
-             }
-         }
-
-         if (sh->mpm_huad_ctx_ts != NULL) {
-             if (sh->mpm_huad_ctx_ts->pattern_cnt == 0) {
-                 MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_huad_ctx_ts);
-                 sh->mpm_huad_ctx_ts = NULL;
-             } else {
-                 if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
-                     if (mpm_table[sh->mpm_huad_ctx_ts->mpm_type].Prepare != NULL)
-                         mpm_table[sh->mpm_huad_ctx_ts->mpm_type].Prepare(sh->mpm_huad_ctx_ts);
-                 }
-             }
-         }
-
-         if (sh->mpm_hhhd_ctx_ts != NULL) {
-             if (sh->mpm_hhhd_ctx_ts->pattern_cnt == 0) {
-                 MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hhhd_ctx_ts);
-                 sh->mpm_hhhd_ctx_ts = NULL;
-             } else {
-                 if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
-                     if (mpm_table[sh->mpm_hhhd_ctx_ts->mpm_type].Prepare != NULL)
-                         mpm_table[sh->mpm_hhhd_ctx_ts->mpm_type].Prepare(sh->mpm_hhhd_ctx_ts);
-                 }
-             }
-         }
-
-         if (sh->mpm_hrhhd_ctx_ts != NULL) {
-             if (sh->mpm_hrhhd_ctx_ts->pattern_cnt == 0) {
-                 MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hrhhd_ctx_ts);
-                 sh->mpm_hrhhd_ctx_ts = NULL;
-             } else {
-                 if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
-                     if (mpm_table[sh->mpm_hrhhd_ctx_ts->mpm_type].Prepare != NULL)
-                         mpm_table[sh->mpm_hrhhd_ctx_ts->mpm_type].Prepare(sh->mpm_hrhhd_ctx_ts);
-                 }
-             }
-         }
-
-         if (sh->mpm_dnsquery_ctx_ts != NULL) {
-             if (sh->mpm_dnsquery_ctx_ts->pattern_cnt == 0) {
-                 MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_dnsquery_ctx_ts);
-                 sh->mpm_dnsquery_ctx_ts = NULL;
-             } else {
-                 if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
-                     if (mpm_table[sh->mpm_dnsquery_ctx_ts->mpm_type].Prepare != NULL)
-                         mpm_table[sh->mpm_dnsquery_ctx_ts->mpm_type].Prepare(sh->mpm_dnsquery_ctx_ts);
-                 }
-             }
-         }
-        //} /* if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) */
+        if (sh->mpm_proto_tcp_ctx_ts != NULL) {
+            if (sh->mpm_proto_tcp_ctx_ts->pattern_cnt == 0) {
+                MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_proto_tcp_ctx_ts);
+                sh->mpm_proto_tcp_ctx_ts = NULL;
+            } else {
+                if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
+                    if (mpm_table[sh->mpm_proto_tcp_ctx_ts->mpm_type].Prepare != NULL) {
+                        mpm_table[sh->mpm_proto_tcp_ctx_ts->mpm_type].
+                            Prepare(sh->mpm_proto_tcp_ctx_ts);
+                    }
+                }
+            }
+        }
+        if (sh->mpm_proto_tcp_ctx_tc != NULL) {
+            if (sh->mpm_proto_tcp_ctx_tc->pattern_cnt == 0) {
+                MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_proto_tcp_ctx_tc);
+                sh->mpm_proto_tcp_ctx_tc = NULL;
+            } else {
+                if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
+                    if (mpm_table[sh->mpm_proto_tcp_ctx_tc->mpm_type].Prepare != NULL) {
+                        mpm_table[sh->mpm_proto_tcp_ctx_tc->mpm_type].
+                            Prepare(sh->mpm_proto_tcp_ctx_tc);
+                    }
+                }
+            }
+        }
+
+        if (sh->mpm_proto_udp_ctx_ts != NULL) {
+            if (sh->mpm_proto_udp_ctx_ts->pattern_cnt == 0) {
+                MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_proto_udp_ctx_ts);
+                sh->mpm_proto_udp_ctx_ts = NULL;
+            } else {
+                if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
+                    if (mpm_table[sh->mpm_proto_udp_ctx_ts->mpm_type].Prepare != NULL) {
+                        mpm_table[sh->mpm_proto_udp_ctx_ts->mpm_type].
+                            Prepare(sh->mpm_proto_udp_ctx_ts);
+                    }
+                }
+            }
+        }
+        if (sh->mpm_proto_udp_ctx_tc != NULL) {
+            if (sh->mpm_proto_udp_ctx_tc->pattern_cnt == 0) {
+                MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_proto_udp_ctx_tc);
+                sh->mpm_proto_udp_ctx_tc = NULL;
+            } else {
+                if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
+                    if (mpm_table[sh->mpm_proto_udp_ctx_tc->mpm_type].Prepare != NULL) {
+                        mpm_table[sh->mpm_proto_udp_ctx_tc->mpm_type].
+                            Prepare(sh->mpm_proto_udp_ctx_tc);
+                    }
+                }
+            }
+        }
+
+        if (sh->mpm_proto_other_ctx != NULL) {
+            if (sh->mpm_proto_other_ctx->pattern_cnt == 0) {
+                MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_proto_other_ctx);
+                sh->mpm_proto_other_ctx = NULL;
+            } else {
+                if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
+                    if (mpm_table[sh->mpm_proto_other_ctx->mpm_type].Prepare != NULL) {
+                        mpm_table[sh->mpm_proto_other_ctx->mpm_type].
+                            Prepare(sh->mpm_proto_other_ctx);
+                    }
+                }
+            }
+        }
+
+        if (sh->mpm_stream_ctx_ts != NULL) {
+            if (sh->mpm_stream_ctx_ts->pattern_cnt == 0) {
+                MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_stream_ctx_ts);
+                sh->mpm_stream_ctx_ts = NULL;
+            } else {
+                if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
+                    if (mpm_table[sh->mpm_stream_ctx_ts->mpm_type].Prepare != NULL)
+                        mpm_table[sh->mpm_stream_ctx_ts->mpm_type].Prepare(sh->mpm_stream_ctx_ts);
+                }
+            }
+        }
+        if (sh->mpm_stream_ctx_tc != NULL) {
+            if (sh->mpm_stream_ctx_tc->pattern_cnt == 0) {
+                MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_stream_ctx_tc);
+                sh->mpm_stream_ctx_tc = NULL;
+            } else {
+                if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
+                    if (mpm_table[sh->mpm_stream_ctx_tc->mpm_type].Prepare != NULL)
+                        mpm_table[sh->mpm_stream_ctx_tc->mpm_type].Prepare(sh->mpm_stream_ctx_tc);
+                }
+            }
+        }
+
+        if (sh->mpm_uri_ctx_ts != NULL) {
+            if (sh->mpm_uri_ctx_ts->pattern_cnt == 0) {
+                MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_uri_ctx_ts);
+                sh->mpm_uri_ctx_ts = NULL;
+            } else {
+                if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
+                    if (mpm_table[sh->mpm_uri_ctx_ts->mpm_type].Prepare != NULL)
+                        mpm_table[sh->mpm_uri_ctx_ts->mpm_type].Prepare(sh->mpm_uri_ctx_ts);
+                }
+            }
+        }
+
+        if (sh->mpm_hcbd_ctx_ts != NULL) {
+            if (sh->mpm_hcbd_ctx_ts->pattern_cnt == 0) {
+                MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hcbd_ctx_ts);
+                sh->mpm_hcbd_ctx_ts = NULL;
+            } else {
+                if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
+                    if (mpm_table[sh->mpm_hcbd_ctx_ts->mpm_type].Prepare != NULL)
+                        mpm_table[sh->mpm_hcbd_ctx_ts->mpm_type].Prepare(sh->mpm_hcbd_ctx_ts);
+                }
+            }
+        }
+
+        if (sh->mpm_hsbd_ctx_tc != NULL) {
+            if (sh->mpm_hsbd_ctx_tc->pattern_cnt == 0) {
+                MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hsbd_ctx_tc);
+                sh->mpm_hsbd_ctx_tc = NULL;
+            } else {
+                if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
+                    if (mpm_table[sh->mpm_hsbd_ctx_tc->mpm_type].Prepare != NULL)
+                        mpm_table[sh->mpm_hsbd_ctx_tc->mpm_type].Prepare(sh->mpm_hsbd_ctx_tc);
+                }
+            }
+        }
+
+        if (sh->mpm_smtp_filedata_ctx_ts != NULL) {
+            if (sh->mpm_smtp_filedata_ctx_ts->pattern_cnt == 0) {
+                MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_smtp_filedata_ctx_ts);
+                sh->mpm_smtp_filedata_ctx_ts = NULL;
+            } else {
+                if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
+                    if (mpm_table[sh->mpm_smtp_filedata_ctx_ts->mpm_type].Prepare != NULL) {
+                        mpm_table[sh->mpm_smtp_filedata_ctx_ts->mpm_type].Prepare(sh->mpm_smtp_filedata_ctx_ts);
+                    }
+                }
+            }
+        }
+
+        if (sh->mpm_hhd_ctx_ts != NULL) {
+            if (sh->mpm_hhd_ctx_ts->pattern_cnt == 0) {
+                MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hhd_ctx_ts);
+                sh->mpm_hhd_ctx_ts = NULL;
+            } else {
+                if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
+                    if (mpm_table[sh->mpm_hhd_ctx_ts->mpm_type].Prepare != NULL)
+                        mpm_table[sh->mpm_hhd_ctx_ts->mpm_type].Prepare(sh->mpm_hhd_ctx_ts);
+                }
+            }
+        }
+        if (sh->mpm_hhd_ctx_tc != NULL) {
+            if (sh->mpm_hhd_ctx_tc->pattern_cnt == 0) {
+                MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hhd_ctx_tc);
+                sh->mpm_hhd_ctx_tc = NULL;
+            } else {
+                if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
+                    if (mpm_table[sh->mpm_hhd_ctx_tc->mpm_type].Prepare != NULL)
+                        mpm_table[sh->mpm_hhd_ctx_tc->mpm_type].Prepare(sh->mpm_hhd_ctx_tc);
+                }
+            }
+        }
+
+        if (sh->mpm_hrhd_ctx_ts != NULL) {
+            if (sh->mpm_hrhd_ctx_ts->pattern_cnt == 0) {
+                MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hrhd_ctx_ts);
+                sh->mpm_hrhd_ctx_ts = NULL;
+            } else {
+                if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
+                    if (mpm_table[sh->mpm_hrhd_ctx_ts->mpm_type].Prepare != NULL)
+                        mpm_table[sh->mpm_hrhd_ctx_ts->mpm_type].Prepare(sh->mpm_hrhd_ctx_ts);
+                }
+            }
+        }
+        if (sh->mpm_hrhd_ctx_tc != NULL) {
+            if (sh->mpm_hrhd_ctx_tc->pattern_cnt == 0) {
+                MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hrhd_ctx_tc);
+                sh->mpm_hrhd_ctx_tc = NULL;
+            } else {
+                if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
+                    if (mpm_table[sh->mpm_hrhd_ctx_tc->mpm_type].Prepare != NULL)
+                        mpm_table[sh->mpm_hrhd_ctx_tc->mpm_type].Prepare(sh->mpm_hrhd_ctx_tc);
+                }
+            }
+        }
+
+        if (sh->mpm_hmd_ctx_ts != NULL) {
+            if (sh->mpm_hmd_ctx_ts->pattern_cnt == 0) {
+                MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hmd_ctx_ts);
+                sh->mpm_hmd_ctx_ts = NULL;
+            } else {
+                if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
+                    if (mpm_table[sh->mpm_hmd_ctx_ts->mpm_type].Prepare != NULL)
+                        mpm_table[sh->mpm_hmd_ctx_ts->mpm_type].Prepare(sh->mpm_hmd_ctx_ts);
+                }
+            }
+        }
+
+        if (sh->mpm_hcd_ctx_ts != NULL) {
+            if (sh->mpm_hcd_ctx_ts->pattern_cnt == 0) {
+                MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hcd_ctx_ts);
+                sh->mpm_hcd_ctx_ts = NULL;
+            } else {
+                if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
+                    if (mpm_table[sh->mpm_hcd_ctx_ts->mpm_type].Prepare != NULL)
+                        mpm_table[sh->mpm_hcd_ctx_ts->mpm_type].Prepare(sh->mpm_hcd_ctx_ts);
+                }
+            }
+        }
+        if (sh->mpm_hcd_ctx_tc != NULL) {
+            if (sh->mpm_hcd_ctx_tc->pattern_cnt == 0) {
+                MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hcd_ctx_tc);
+                sh->mpm_hcd_ctx_tc = NULL;
+            } else {
+                if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
+                    if (mpm_table[sh->mpm_hcd_ctx_tc->mpm_type].Prepare != NULL)
+                        mpm_table[sh->mpm_hcd_ctx_tc->mpm_type].Prepare(sh->mpm_hcd_ctx_tc);
+                }
+            }
+        }
+
+        if (sh->mpm_hrud_ctx_ts != NULL) {
+            if (sh->mpm_hrud_ctx_ts->pattern_cnt == 0) {
+                MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hrud_ctx_ts);
+                sh->mpm_hrud_ctx_ts = NULL;
+            } else {
+                if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
+                    if (mpm_table[sh->mpm_hrud_ctx_ts->mpm_type].Prepare != NULL)
+                        mpm_table[sh->mpm_hrud_ctx_ts->mpm_type].Prepare(sh->mpm_hrud_ctx_ts);
+                }
+            }
+        }
+
+        if (sh->mpm_hsmd_ctx_tc != NULL) {
+            if (sh->mpm_hsmd_ctx_tc->pattern_cnt == 0) {
+                MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hsmd_ctx_tc);
+                sh->mpm_hsmd_ctx_tc = NULL;
+            } else {
+                if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
+                    if (mpm_table[sh->mpm_hsmd_ctx_tc->mpm_type].Prepare != NULL)
+                        mpm_table[sh->mpm_hsmd_ctx_tc->mpm_type].Prepare(sh->mpm_hsmd_ctx_tc);
+                }
+            }
+        }
+
+        if (sh->mpm_hscd_ctx_tc != NULL) {
+            if (sh->mpm_hscd_ctx_tc->pattern_cnt == 0) {
+                MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hscd_ctx_tc);
+                sh->mpm_hscd_ctx_tc = NULL;
+            } else {
+                if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
+                    if (mpm_table[sh->mpm_hscd_ctx_tc->mpm_type].Prepare != NULL)
+                        mpm_table[sh->mpm_hscd_ctx_tc->mpm_type].Prepare(sh->mpm_hscd_ctx_tc);
+                }
+            }
+        }
+
+        if (sh->mpm_huad_ctx_ts != NULL) {
+            if (sh->mpm_huad_ctx_ts->pattern_cnt == 0) {
+                MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_huad_ctx_ts);
+                sh->mpm_huad_ctx_ts = NULL;
+            } else {
+                if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
+                    if (mpm_table[sh->mpm_huad_ctx_ts->mpm_type].Prepare != NULL)
+                        mpm_table[sh->mpm_huad_ctx_ts->mpm_type].Prepare(sh->mpm_huad_ctx_ts);
+                }
+            }
+        }
+
+        if (sh->mpm_hhhd_ctx_ts != NULL) {
+            if (sh->mpm_hhhd_ctx_ts->pattern_cnt == 0) {
+                MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hhhd_ctx_ts);
+                sh->mpm_hhhd_ctx_ts = NULL;
+            } else {
+                if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
+                    if (mpm_table[sh->mpm_hhhd_ctx_ts->mpm_type].Prepare != NULL)
+                        mpm_table[sh->mpm_hhhd_ctx_ts->mpm_type].Prepare(sh->mpm_hhhd_ctx_ts);
+                }
+            }
+        }
+
+        if (sh->mpm_hrhhd_ctx_ts != NULL) {
+            if (sh->mpm_hrhhd_ctx_ts->pattern_cnt == 0) {
+                MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hrhhd_ctx_ts);
+                sh->mpm_hrhhd_ctx_ts = NULL;
+            } else {
+                if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
+                    if (mpm_table[sh->mpm_hrhhd_ctx_ts->mpm_type].Prepare != NULL)
+                        mpm_table[sh->mpm_hrhhd_ctx_ts->mpm_type].Prepare(sh->mpm_hrhhd_ctx_ts);
+                }
+            }
+        }
+
+        if (sh->mpm_dnsquery_ctx_ts != NULL) {
+            if (sh->mpm_dnsquery_ctx_ts->pattern_cnt == 0) {
+                MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_dnsquery_ctx_ts);
+                sh->mpm_dnsquery_ctx_ts = NULL;
+            } else {
+                if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) {
+                    if (mpm_table[sh->mpm_dnsquery_ctx_ts->mpm_type].Prepare != NULL)
+                        mpm_table[sh->mpm_dnsquery_ctx_ts->mpm_type].Prepare(sh->mpm_dnsquery_ctx_ts);
+                }
+            }
+        }
     } else {
         MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_proto_other_ctx);
         sh->mpm_proto_other_ctx = NULL;
diff --git a/framework/src/suricata/src/detect-engine-siggroup.c b/framework/src/suricata/src/detect-engine-siggroup.c
index 02602d90..89e0eb79 100644
--- a/framework/src/suricata/src/detect-engine-siggroup.c
+++ b/framework/src/suricata/src/detect-engine-siggroup.c
@@ -608,7 +608,7 @@ uint32_t SigGroupHeadHashFunc(HashListTable *ht, void *data, uint16_t datalen)
     uint32_t hash = 0;
     uint32_t b = 0;
 
-    SCLogDebug("hashing sgh %p (mpm_content_maxlen %u)", sgh, sgh->mpm_content_maxlen);
+    SCLogDebug("hashing sgh %p (mpm_content_minlen %u)", sgh, sgh->mpm_content_minlen);
 
     for (b = 0; b < sgh->init->sig_size; b++)
         hash += sgh->init->sig_array[b];
@@ -998,6 +998,17 @@ void SigGroupHeadFreeMpmArrays(DetectEngineCtx *de_ctx)
     return;
 }
 
+static uint16_t SignatureGetMpmPatternLen(Signature *s, int list)
+{
+    if (s->sm_lists[list] != NULL && s->mpm_sm != NULL &&
+        SigMatchListSMBelongsTo(s, s->mpm_sm) == list)
+    {
+        DetectContentData *cd = (DetectContentData *)s->mpm_sm->ctx;
+        return cd->content_len;
+    }
+    return 0;
+}
+
 /**
  * \brief Add a Signature to a SigGroupHead.
  *
@@ -1025,28 +1036,18 @@ int SigGroupHeadAppendSig(DetectEngineCtx *de_ctx, SigGroupHead **sgh,
     /* enable the sig in the bitarray */
     (*sgh)->init->sig_array[s->num / 8] |= 1 << (s->num % 8);
 
-    /* update maxlen for mpm */
+    /* update minlen for mpm */
     if (s->sm_lists[DETECT_SM_LIST_PMATCH] != NULL) {
         /* check with the precalculated values from the sig */
-        if (s->mpm_content_maxlen > 0) {
-            if ((*sgh)->mpm_content_maxlen == 0)
-                (*sgh)->mpm_content_maxlen = s->mpm_content_maxlen;
+        uint16_t mpm_content_minlen = SignatureGetMpmPatternLen(s, DETECT_SM_LIST_PMATCH);
+        if (mpm_content_minlen > 0) {
+            if ((*sgh)->mpm_content_minlen == 0)
+                (*sgh)->mpm_content_minlen = mpm_content_minlen;
 
-            if ((*sgh)->mpm_content_maxlen > s->mpm_content_maxlen)
-                (*sgh)->mpm_content_maxlen = s->mpm_content_maxlen;
+            if ((*sgh)->mpm_content_minlen > mpm_content_minlen)
+                (*sgh)->mpm_content_minlen = mpm_content_minlen;
 
-            SCLogDebug("(%p)->mpm_content_maxlen %u", *sgh, (*sgh)->mpm_content_maxlen);
-        }
-    }
-    if (s->sm_lists[DETECT_SM_LIST_UMATCH] != NULL) {
-        if (s->mpm_uricontent_maxlen > 0) {
-            if ((*sgh)->mpm_uricontent_maxlen == 0)
-                (*sgh)->mpm_uricontent_maxlen = s->mpm_uricontent_maxlen;
-
-            if ((*sgh)->mpm_uricontent_maxlen > s->mpm_uricontent_maxlen)
-                (*sgh)->mpm_uricontent_maxlen = s->mpm_uricontent_maxlen;
-
-            SCLogDebug("(%p)->mpm_uricontent_maxlen %u", *sgh, (*sgh)->mpm_uricontent_maxlen);
+            SCLogDebug("(%p)->mpm_content_minlen %u", *sgh, (*sgh)->mpm_content_minlen);
         }
     }
     return 0;
@@ -1103,23 +1104,16 @@ int SigGroupHeadCopySigs(DetectEngineCtx *de_ctx, SigGroupHead *src, SigGroupHea
     for (idx = 0; idx < src->init->sig_size; idx++)
         (*dst)->init->sig_array[idx] = (*dst)->init->sig_array[idx] | src->init->sig_array[idx];
 
-    if (src->mpm_content_maxlen != 0) {
-        if ((*dst)->mpm_content_maxlen == 0)
-            (*dst)->mpm_content_maxlen = src->mpm_content_maxlen;
-
-        if ((*dst)->mpm_content_maxlen > src->mpm_content_maxlen)
-            (*dst)->mpm_content_maxlen = src->mpm_content_maxlen;
+    if (src->mpm_content_minlen != 0) {
+        if ((*dst)->mpm_content_minlen == 0)
+            (*dst)->mpm_content_minlen = src->mpm_content_minlen;
 
-        SCLogDebug("src (%p)->mpm_content_maxlen %u", src, src->mpm_content_maxlen);
-        SCLogDebug("dst (%p)->mpm_content_maxlen %u", (*dst), (*dst)->mpm_content_maxlen);
-        BUG_ON((*dst)->mpm_content_maxlen == 0);
-    }
-    if (src->mpm_uricontent_maxlen != 0) {
-        if ((*dst)->mpm_uricontent_maxlen == 0)
-            (*dst)->mpm_uricontent_maxlen = src->mpm_uricontent_maxlen;
+        if ((*dst)->mpm_content_minlen > src->mpm_content_minlen)
+            (*dst)->mpm_content_minlen = src->mpm_content_minlen;
 
-        if ((*dst)->mpm_uricontent_maxlen > src->mpm_uricontent_maxlen)
-            (*dst)->mpm_uricontent_maxlen = src->mpm_uricontent_maxlen;
+        SCLogDebug("src (%p)->mpm_content_minlen %u", src, src->mpm_content_minlen);
+        SCLogDebug("dst (%p)->mpm_content_minlen %u", (*dst), (*dst)->mpm_content_minlen);
+        BUG_ON((*dst)->mpm_content_minlen == 0);
     }
     return 0;
 
@@ -1606,6 +1600,42 @@ void SigGroupHeadSetFilemagicFlag(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
 }
 
 /**
+ *  \brief Get size of the shortest mpm pattern.
+ *
+ *  \param de_ctx detection engine ctx for the signatures
+ *  \param sgh sig group head to set the flag in
+ *  \param list sm_list to consider
+ */
+uint16_t SigGroupHeadGetMinMpmSize(DetectEngineCtx *de_ctx,
+                                   SigGroupHead *sgh, int list)
+{
+    Signature *s = NULL;
+    uint32_t sig = 0;
+    uint16_t min = USHRT_MAX;
+
+    if (sgh == NULL)
+        return 0;
+
+    for (sig = 0; sig < sgh->sig_cnt; sig++) {
+        s = sgh->match_array[sig];
+        if (s == NULL)
+            continue;
+
+        uint16_t mpm_content_minlen = SignatureGetMpmPatternLen(s, DETECT_SM_LIST_PMATCH);
+        if (mpm_content_minlen > 0) {
+            if (mpm_content_minlen < min)
+                min = mpm_content_minlen;
+            SCLogDebug("mpm_content_minlen %u", mpm_content_minlen);
+        }
+    }
+
+    if (min == USHRT_MAX)
+        min = 0;
+    SCLogDebug("min mpm size %u", min);
+    return min;
+}
+
+/**
  *  \brief Set the need size flag in the sgh.
  *
  *  \param de_ctx detection engine ctx for the signatures
diff --git a/framework/src/suricata/src/detect-engine-siggroup.h b/framework/src/suricata/src/detect-engine-siggroup.h
index 829b0cef..cd6810a1 100644
--- a/framework/src/suricata/src/detect-engine-siggroup.h
+++ b/framework/src/suricata/src/detect-engine-siggroup.h
@@ -88,6 +88,8 @@ void SigGroupHeadSetFilemagicFlag(DetectEngineCtx *, SigGroupHead *);
 void SigGroupHeadSetFilestoreCount(DetectEngineCtx *, SigGroupHead *);
 void SigGroupHeadSetFileMd5Flag(DetectEngineCtx *, SigGroupHead *);
 void SigGroupHeadSetFilesizeFlag(DetectEngineCtx *, SigGroupHead *);
+uint16_t SigGroupHeadGetMinMpmSize(DetectEngineCtx *de_ctx,
+                                   SigGroupHead *sgh, int list);
 
 int SigGroupHeadBuildNonMpmArray(DetectEngineCtx *de_ctx, SigGroupHead *sgh);
 
diff --git a/framework/src/suricata/src/detect-engine-threshold.c b/framework/src/suricata/src/detect-engine-threshold.c
index 6c6d1371..2d37e55e 100644
--- a/framework/src/suricata/src/detect-engine-threshold.c
+++ b/framework/src/suricata/src/detect-engine-threshold.c
@@ -216,11 +216,11 @@ int ThresholdHandlePacketSuppress(Packet *p, DetectThresholdData *td, uint32_t s
     switch (td->track) {
         case TRACK_DST:
             m = DetectAddressLookupInHead(&td->addrs, &p->dst);
-            SCLogInfo("TRACK_DST");
+            SCLogDebug("TRACK_DST");
             break;
         case TRACK_SRC:
             m = DetectAddressLookupInHead(&td->addrs, &p->src);
-            SCLogInfo("TRACK_SRC");
+            SCLogDebug("TRACK_SRC");
             break;
         /* suppress if either src or dst is a match on the suppress
          * address list */
diff --git a/framework/src/suricata/src/detect-engine.c b/framework/src/suricata/src/detect-engine.c
index c6e1a83f..431f4b2a 100644
--- a/framework/src/suricata/src/detect-engine.c
+++ b/framework/src/suricata/src/detect-engine.c
@@ -2340,7 +2340,7 @@ static int reloads = 0;
  *  \retval -1 error
  *  \retval 0 ok
  */
-int DetectEngineReload(const char *filename)
+int DetectEngineReload(const char *filename, SCInstance *suri)
 {
     DetectEngineCtx *new_de_ctx = NULL;
     DetectEngineCtx *old_de_ctx = NULL;
@@ -2377,7 +2377,8 @@ int DetectEngineReload(const char *filename)
         DetectEngineDeReference(&old_de_ctx);
         return -1;
     }
-    if (SigLoadSignatures(new_de_ctx, NULL, 0) != 0) {
+    if (SigLoadSignatures(new_de_ctx,
+                          suri->sig_file, suri->sig_file_exclusive) != 0) {
         DetectEngineCtxFree(new_de_ctx);
         DetectEngineDeReference(&old_de_ctx);
         return -1;
diff --git a/framework/src/suricata/src/detect-engine.h b/framework/src/suricata/src/detect-engine.h
index ce486213..7b621710 100644
--- a/framework/src/suricata/src/detect-engine.h
+++ b/framework/src/suricata/src/detect-engine.h
@@ -75,7 +75,7 @@ void DetectEnginePruneFreeList(void);
 int DetectEngineMoveToFreeList(DetectEngineCtx *de_ctx);
 DetectEngineCtx *DetectEngineReference(DetectEngineCtx *);
 void DetectEngineDeReference(DetectEngineCtx **de_ctx);
-int DetectEngineReload(const char *filename);
+int DetectEngineReload(const char *filename, SCInstance *suri);
 int DetectEngineEnabled(void);
 int DetectEngineMTApply(void);
 int DetectEngineMultiTenantEnabled(void);
diff --git a/framework/src/suricata/src/detect-lua-extensions.c b/framework/src/suricata/src/detect-lua-extensions.c
index 020c886d..ae9b4e14 100644
--- a/framework/src/suricata/src/detect-lua-extensions.c
+++ b/framework/src/suricata/src/detect-lua-extensions.c
@@ -67,6 +67,7 @@
 #include "util-lua-http.h"
 #include "util-lua-dns.h"
 #include "util-lua-tls.h"
+#include "util-lua-ssh.h"
 
 static const char luaext_key_ld[] = "suricata:luajitdata";
 static const char luaext_key_det_ctx[] = "suricata:det_ctx";
@@ -619,6 +620,7 @@ int LuaRegisterExtensions(lua_State *lua_state)
     LuaRegisterHttpFunctions(lua_state);
     LuaRegisterDnsFunctions(lua_state);
     LuaRegisterTlsFunctions(lua_state);
+    LuaRegisterSshFunctions(lua_state);
     return 0;
 }
 
diff --git a/framework/src/suricata/src/detect-lua.c b/framework/src/suricata/src/detect-lua.c
index d1f2cd00..a3eead4a 100644
--- a/framework/src/suricata/src/detect-lua.c
+++ b/framework/src/suricata/src/detect-lua.c
@@ -166,6 +166,8 @@ void DetectLuaRegister(void)
 
 #define DATATYPE_TLS                        (1<<18)
 
+#define DATATYPE_SSH                        (1<<19)
+
 #ifdef HAVE_LUAJIT
 static void *LuaStatePoolAlloc(void)
 {
@@ -190,12 +192,18 @@ int DetectLuajitSetupStatesPool(int num, int reloads)
     pthread_mutex_lock(&luajit_states_lock);
 
     if (luajit_states == NULL) {
-        int cnt = 0;
-        char *conf_val = NULL;
+        intmax_t cnt = 0;
+        ConfNode *denode = NULL;
+        ConfNode *decnf = ConfGetNode("detect-engine");
+        if (decnf != NULL) {
+            TAILQ_FOREACH(denode, &decnf->head, next) {
+                if (strcmp(denode->val, "luajit-states") == 0) {
+                    ConfGetChildValueInt(denode, "luajit-states", &cnt);
+                }
+            }
+        }
 
-        if ((ConfGet("detect-engine.luajit-states", &conf_val)) == 1) {
-            cnt = (int)atoi(conf_val);
-        } else {
+        if (cnt == 0) {
             int cpus = UtilCpuGetNumProcessorsOnline();
             if (cpus == 0) {
                 cpus = 10;
@@ -1008,6 +1016,12 @@ static int DetectLuaSetupPrime(DetectEngineCtx *de_ctx, DetectLuaData *ld)
 
             ld->flags |= DATATYPE_TLS;
 
+        } else if (strncmp(k, "ssh", 3) == 0 && strcmp(v, "true") == 0) {
+
+            ld->alproto = ALPROTO_SSH;
+
+            ld->flags |= DATATYPE_SSH;
+
         } else {
             SCLogError(SC_ERR_LUA_ERROR, "unsupported data type %s", k);
             goto error;
@@ -1105,6 +1119,8 @@ static int DetectLuaSetup (DetectEngineCtx *de_ctx, Signature *s, char *str)
         }
     } else if (luajit->alproto == ALPROTO_TLS) {
         SigMatchAppendSMToList(s, sm, DETECT_SM_LIST_AMATCH);
+    } else if (luajit->alproto == ALPROTO_SSH) {
+        SigMatchAppendSMToList(s, sm, DETECT_SM_LIST_AMATCH);
     } else {
         SCLogError(SC_ERR_LUA_ERROR, "luajit can't be used with protocol %s",
                    AppLayerGetProtoName(luajit->alproto));
diff --git a/framework/src/suricata/src/detect-parse.c b/framework/src/suricata/src/detect-parse.c
index fedfebe8..9ac53e03 100644
--- a/framework/src/suricata/src/detect-parse.c
+++ b/framework/src/suricata/src/detect-parse.c
@@ -1345,7 +1345,6 @@ int SigValidate(DetectEngineCtx *de_ctx, Signature *s)
 static Signature *SigInitHelper(DetectEngineCtx *de_ctx, char *sigstr,
                                 uint8_t dir)
 {
-    SigMatch *sm;
     Signature *sig = SigAlloc();
     if (sig == NULL)
         goto error;
@@ -1390,42 +1389,6 @@ static Signature *SigInitHelper(DetectEngineCtx *de_ctx, char *sigstr,
     if (DetectAppLayerEventPrepare(sig) < 0)
         goto error;
 
-    /* set mpm_content_len */
-
-    /* determine the length of the longest pattern in the sig */
-    if (sig->sm_lists[DETECT_SM_LIST_PMATCH] != NULL) {
-        sig->mpm_content_maxlen = 0;
-
-        for (sm = sig->sm_lists[DETECT_SM_LIST_PMATCH]; sm != NULL; sm = sm->next) {
-            if (sm->type == DETECT_CONTENT) {
-                DetectContentData *cd = (DetectContentData *)sm->ctx;
-                 if (cd == NULL)
-                    continue;
-
-                if (sig->mpm_content_maxlen == 0)
-                    sig->mpm_content_maxlen = cd->content_len;
-                if (sig->mpm_content_maxlen < cd->content_len)
-                    sig->mpm_content_maxlen = cd->content_len;
-            }
-        }
-    }
-    if (sig->sm_lists[DETECT_SM_LIST_UMATCH] != NULL) {
-        sig->mpm_uricontent_maxlen = 0;
-
-        for (sm = sig->sm_lists[DETECT_SM_LIST_UMATCH]; sm != NULL; sm = sm->next) {
-            if (sm->type == DETECT_CONTENT) {
-                DetectContentData *ud = (DetectContentData *)sm->ctx;
-                if (ud == NULL)
-                    continue;
-
-                if (sig->mpm_uricontent_maxlen == 0)
-                    sig->mpm_uricontent_maxlen = ud->content_len;
-                if (sig->mpm_uricontent_maxlen < ud->content_len)
-                    sig->mpm_uricontent_maxlen = ud->content_len;
-            }
-        }
-    }
-
     /* set the packet and app layer flags, but only if the
      * app layer flag wasn't already set in which case we
      * only consider the app layer */
@@ -3268,16 +3231,6 @@ int SigParseTestMpm01 (void)
         goto end;
     }
 
-    if (sig->mpm_content_maxlen != 4) {
-        printf("mpm content max len %"PRIu16", expected 4: ", sig->mpm_content_maxlen);
-        goto end;
-    }
-
-    if (sig->mpm_uricontent_maxlen != 0) {
-        printf("mpm uricontent max len %"PRIu16", expected 0: ", sig->mpm_uricontent_maxlen);
-        goto end;
-    }
-
     result = 1;
 end:
     if (sig != NULL)
@@ -3309,16 +3262,6 @@ int SigParseTestMpm02 (void)
         goto end;
     }
 
-    if (sig->mpm_content_maxlen != 6) {
-        printf("mpm content max len %"PRIu16", expected 6: ", sig->mpm_content_maxlen);
-        goto end;
-    }
-
-    if (sig->mpm_uricontent_maxlen != 0) {
-        printf("mpm uricontent max len %"PRIu16", expected 0: ", sig->mpm_uricontent_maxlen);
-        goto end;
-    }
-
     result = 1;
 end:
     if (sig != NULL)
diff --git a/framework/src/suricata/src/detect-pcre.c b/framework/src/suricata/src/detect-pcre.c
index 86f3d169..d69f1da6 100644
--- a/framework/src/suricata/src/detect-pcre.c
+++ b/framework/src/suricata/src/detect-pcre.c
@@ -274,6 +274,26 @@ static int DetectPcreSetList(int list, int set)
     return set;
 }
 
+static int DetectPcreHasUpperCase(const char *re)
+{
+    size_t len = strlen(re);
+    int is_meta = 0;
+
+    for (size_t i = 0; i < len; i++) {
+        if (is_meta) {
+            is_meta = 0;
+        }
+        else if (re[i] == '\\') {
+            is_meta = 1;
+        }
+        else if (isupper((unsigned char)re[i])) {
+            return 1;
+        }
+    }
+
+    return 0;
+}
+
 static DetectPcreData *DetectPcreParse (DetectEngineCtx *de_ctx, char *regexstr, int *sm_list)
 {
     int ec;
@@ -290,8 +310,6 @@ static DetectPcreData *DetectPcreParse (DetectEngineCtx *de_ctx, char *regexstr,
     char re[slen], op_str[64] = "";
     uint16_t pos = 0;
     uint8_t negate = 0;
-    uint16_t re_len = 0;
-    uint32_t u = 0;
 
     while (pos < slen && isspace((unsigned char)regexstr[pos])) {
         pos++;
@@ -476,18 +494,14 @@ static DetectPcreData *DetectPcreParse (DetectEngineCtx *de_ctx, char *regexstr,
                          "Since the hostname buffer we match against "
                          "is actually lowercase, having a "
                          "nocase is redundant.");
-        } else {
-            re_len = strlen(re);
-            for (u = 0; u < re_len; u++) {
-                if (isupper((unsigned char)re[u])) {
-                    SCLogError(SC_ERR_INVALID_SIGNATURE, "pcre host(\"W\") "
-                               "specified has an uppercase char.  "
-                               "Since the hostname buffer we match against "
-                               "is actually lowercase, please specify an "
-                               "all lowercase based pcre.");
-                    goto error;
-                }
-            }
+        }
+        else if (DetectPcreHasUpperCase(re)) {
+            SCLogError(SC_ERR_INVALID_SIGNATURE, "pcre host(\"W\") "
+                "specified has an uppercase char.  "
+                "Since the hostname buffer we match against "
+                "is actually lowercase, please specify an "
+                "all lowercase based pcre.");
+            goto error;
         }
     }
 
@@ -4053,6 +4067,57 @@ end:
     return result;
 }
 
+/**
+ * \brief Test parsing of pcre's with the W modifier set.
+ */
+static int DetectPcreParseHttpHost(void)
+{
+    int result = 0;
+    DetectPcreData *pd = NULL;
+    int list = DETECT_SM_LIST_NOTSET;
+    DetectEngineCtx *de_ctx = DetectEngineCtxInit();
+
+    if (de_ctx == NULL) {
+        return 0;
+    }
+
+    pd = DetectPcreParse(de_ctx, "/domain\\.com/W", &list);
+    if (pd == NULL) {
+        goto end;
+    }
+    DetectPcreFree(pd);
+
+    list = DETECT_SM_LIST_NOTSET;
+    pd = DetectPcreParse(de_ctx, "/dOmain\\.com/W", &list);
+    if (pd != NULL) {
+        DetectPcreFree(pd);
+        goto end;
+    }
+
+    /* Uppercase meta characters are valid. */
+    list = DETECT_SM_LIST_NOTSET;
+    pd = DetectPcreParse(de_ctx, "/domain\\D+\\.com/W", &list);
+    if (pd == NULL) {
+        goto end;
+    }
+    DetectPcreFree(pd);
+
+    /* This should not parse as the first \ escapes the second \, then
+     * we have a D. */
+    list = DETECT_SM_LIST_NOTSET;
+    pd = DetectPcreParse(de_ctx, "/\\\\Ddomain\\.com/W", &list);
+    if (pd != NULL) {
+        DetectPcreFree(pd);
+        goto end;
+    }
+
+    result = 1;
+
+end:
+    DetectEngineCtxFree(de_ctx);
+    return result;
+}
+
 #endif /* UNITTESTS */
 
 /**
@@ -4121,6 +4186,8 @@ void DetectPcreRegisterTests(void)
     UtRegisterTest("DetectPcreFlowvarCapture02 -- capture for http_header", DetectPcreFlowvarCapture02, 1);
     UtRegisterTest("DetectPcreFlowvarCapture03 -- capture for http_header", DetectPcreFlowvarCapture03, 1);
 
+    UtRegisterTest("DetectPcreParseHttpHost", DetectPcreParseHttpHost, 1);
+
 #endif /* UNITTESTS */
 }
 
diff --git a/framework/src/suricata/src/detect-template-buffer.c b/framework/src/suricata/src/detect-template-buffer.c
index 64f948c2..d9f9aa67 100644
--- a/framework/src/suricata/src/detect-template-buffer.c
+++ b/framework/src/suricata/src/detect-template-buffer.c
@@ -21,6 +21,7 @@
  */
 
 #include "suricata-common.h"
+#include "conf.h"
 #include "detect.h"
 #include "app-layer-template.h"
 
@@ -29,6 +30,10 @@ static void DetectTemplateBufferRegisterTests(void);
 
 void DetectTemplateBufferRegister(void)
 {
+    if (ConfGetNode("app-layer.protocols.template") == NULL) {
+        return;
+    }
+
     sigmatch_table[DETECT_AL_TEMPLATE_BUFFER].name = "template_buffer";
     sigmatch_table[DETECT_AL_TEMPLATE_BUFFER].desc =
         "Template content modififier to match on the template buffers";
diff --git a/framework/src/suricata/src/detect-uricontent.c b/framework/src/suricata/src/detect-uricontent.c
index a22479b0..65d2ef94 100644
--- a/framework/src/suricata/src/detect-uricontent.c
+++ b/framework/src/suricata/src/detect-uricontent.c
@@ -216,13 +216,13 @@ static inline int DoDetectAppLayerUricontentMatch (DetectEngineThreadCtx *det_ct
 {
     int ret = 0;
     /* run the pattern matcher against the uri */
-    if (det_ctx->sgh->mpm_uricontent_maxlen > uri_len) {
-        SCLogDebug("not searching as pkt payload is smaller than the "
-                "largest uricontent length we need to match");
+    if (det_ctx->sgh->mpm_uricontent_minlen > uri_len) {
+        SCLogDebug("not searching as uri len is smaller than the "
+                   "shortest uricontent length we need to match");
     } else {
-        SCLogDebug("search: (%p, maxlen %" PRIu32 ", sgh->sig_cnt "
-                "%" PRIu32 ")", det_ctx->sgh, det_ctx->sgh->
-                mpm_uricontent_maxlen, det_ctx->sgh->sig_cnt);
+        SCLogDebug("search: (%p, minlen %" PRIu32 ", sgh->sig_cnt "
+                "%" PRIu32 ")", det_ctx->sgh,
+                det_ctx->sgh->mpm_uricontent_minlen, det_ctx->sgh->sig_cnt);
 
         ret += UriPatternSearch(det_ctx, uri, uri_len, flags);
 
diff --git a/framework/src/suricata/src/detect.c b/framework/src/suricata/src/detect.c
index e0b5bfd3..c9a16ead 100644
--- a/framework/src/suricata/src/detect.c
+++ b/framework/src/suricata/src/detect.c
@@ -692,7 +692,7 @@ static StreamMsg *SigMatchSignaturesGetSmsg(Flow *f, Packet *p, uint8_t flags)
 
                 /* if the smsg is bigger than the current packet, we will
                  * process the smsg in a later run */
-                if ((head->seq + head->data_len) > (TCP_GET_SEQ(p) + p->payload_len)) {
+                if (SEQ_GT((head->seq + head->data_len), (TCP_GET_SEQ(p) + p->payload_len))) {
                     SCLogDebug("smsg ends beyond current packet, skipping for now %"PRIu32">%"PRIu32,
                             (head->seq + head->data_len), (TCP_GET_SEQ(p) + p->payload_len));
                     goto end;
@@ -711,7 +711,7 @@ static StreamMsg *SigMatchSignaturesGetSmsg(Flow *f, Packet *p, uint8_t flags)
 
                 /* if the smsg is bigger than the current packet, we will
                  * process the smsg in a later run */
-                if ((head->seq + head->data_len) > (TCP_GET_SEQ(p) + p->payload_len)) {
+                if (SEQ_GT((head->seq + head->data_len), (TCP_GET_SEQ(p) + p->payload_len))) {
                     SCLogDebug("smsg ends beyond current packet, skipping for now %"PRIu32">%"PRIu32,
                             (head->seq + head->data_len), (TCP_GET_SEQ(p) + p->payload_len));
                     goto end;
@@ -1086,8 +1086,8 @@ static inline void DetectMpmPrefilter(DetectEngineCtx *de_ctx,
         }
         if (det_ctx->sgh->flags & SIG_GROUP_HEAD_MPM_PACKET) {
             /* run the multi packet matcher against the payload of the packet */
-            SCLogDebug("search: (%p, maxlen %" PRIu32 ", sgh->sig_cnt %" PRIu32 ")",
-                det_ctx->sgh, det_ctx->sgh->mpm_content_maxlen, det_ctx->sgh->sig_cnt);
+            SCLogDebug("search: (%p, minlen %" PRIu32 ", sgh->sig_cnt %" PRIu32 ")",
+                    det_ctx->sgh, det_ctx->sgh->mpm_content_minlen, det_ctx->sgh->sig_cnt);
 
             PACKET_PROFILING_DETECT_START(p, PROF_DETECT_MPM_PACKET);
             PacketPatternSearch(det_ctx, p);
@@ -3105,15 +3105,15 @@ int CreateGroupedAddrListCmpCnt(DetectAddress *a, DetectAddress *b)
     return 0;
 }
 
-int CreateGroupedAddrListCmpMpmMaxlen(DetectAddress *a, DetectAddress *b)
+int CreateGroupedAddrListCmpMpmMinlen(DetectAddress *a, DetectAddress *b)
 {
     if (a->sh == NULL || b->sh == NULL)
         return 0;
 
-    if (SMALL_MPM(a->sh->mpm_content_maxlen))
+    if (SMALL_MPM(a->sh->mpm_content_minlen))
         return 1;
 
-    if (a->sh->mpm_content_maxlen < b->sh->mpm_content_maxlen)
+    if (a->sh->mpm_content_minlen < b->sh->mpm_content_minlen)
         return 1;
     return 0;
 }
@@ -3139,7 +3139,7 @@ int CreateGroupedAddrList(DetectEngineCtx *de_ctx, DetectAddress *srchead,
     for (gr = srchead; gr != NULL; gr = gr->next) {
         BUG_ON(gr->ip.family == 0 && !(gr->flags & ADDRESS_FLAG_ANY));
 
-        if (SMALL_MPM(gr->sh->mpm_content_maxlen) && unique_groups > 0)
+        if (SMALL_MPM(gr->sh->mpm_content_minlen) && unique_groups > 0)
             unique_groups++;
 
         groups++;
@@ -3280,15 +3280,15 @@ int CreateGroupedPortListCmpCnt(DetectPort *a, DetectPort *b)
     return 0;
 }
 
-int CreateGroupedPortListCmpMpmMaxlen(DetectPort *a, DetectPort *b)
+int CreateGroupedPortListCmpMpmMinlen(DetectPort *a, DetectPort *b)
 {
     if (a->sh == NULL || b->sh == NULL)
         return 0;
 
-    if (SMALL_MPM(a->sh->mpm_content_maxlen))
+    if (SMALL_MPM(a->sh->mpm_content_minlen))
         return 1;
 
-    if (a->sh->mpm_content_maxlen < b->sh->mpm_content_maxlen)
+    if (a->sh->mpm_content_minlen < b->sh->mpm_content_minlen)
         return 1;
 
     return 0;
@@ -3315,7 +3315,7 @@ int CreateGroupedPortList(DetectEngineCtx *de_ctx,HashListTable *port_hash, Dete
         SCLogDebug("hash list gr %p", gr);
         DetectPortPrint(gr);
 
-        if (SMALL_MPM(gr->sh->mpm_content_maxlen) && unique_groups > 0)
+        if (SMALL_MPM(gr->sh->mpm_content_minlen) && unique_groups > 0)
             unique_groups++;
 
         groups++;
@@ -3505,16 +3505,16 @@ int SigAddressPrepareStage2(DetectEngineCtx *de_ctx)
             CreateGroupedAddrList(de_ctx,
                     de_ctx->flow_gh[f].tmp_gh[proto]->ipv4_head, AF_INET,
                     de_ctx->flow_gh[f].src_gh[proto], groups,
-                    CreateGroupedAddrListCmpMpmMaxlen, DetectEngineGetMaxSigId(de_ctx));
+                    CreateGroupedAddrListCmpMpmMinlen, DetectEngineGetMaxSigId(de_ctx));
 
             CreateGroupedAddrList(de_ctx,
                     de_ctx->flow_gh[f].tmp_gh[proto]->ipv6_head, AF_INET6,
                     de_ctx->flow_gh[f].src_gh[proto], groups,
-                    CreateGroupedAddrListCmpMpmMaxlen, DetectEngineGetMaxSigId(de_ctx));
+                    CreateGroupedAddrListCmpMpmMinlen, DetectEngineGetMaxSigId(de_ctx));
             CreateGroupedAddrList(de_ctx,
                     de_ctx->flow_gh[f].tmp_gh[proto]->any_head, AF_UNSPEC,
                     de_ctx->flow_gh[f].src_gh[proto], groups,
-                    CreateGroupedAddrListCmpMpmMaxlen, DetectEngineGetMaxSigId(de_ctx));
+                    CreateGroupedAddrListCmpMpmMinlen, DetectEngineGetMaxSigId(de_ctx));
 
             DetectAddressHeadFree(de_ctx->flow_gh[f].tmp_gh[proto]);
             de_ctx->flow_gh[f].tmp_gh[proto] = NULL;
@@ -3697,7 +3697,8 @@ int BuildDestinationAddressHeads(DetectEngineCtx *de_ctx, DetectAddressHead *hea
          * mind the limits we use. */
         int groups = (flow ? de_ctx->max_uniq_toserver_dst_groups : de_ctx->max_uniq_toclient_dst_groups);
 
-        CreateGroupedAddrList(de_ctx, tmp_gr_list, family, gr->dst_gh, groups, CreateGroupedAddrListCmpMpmMaxlen, max_idx);
+        CreateGroupedAddrList(de_ctx, tmp_gr_list, family, gr->dst_gh, groups,
+                CreateGroupedAddrListCmpMpmMinlen, max_idx);
 
         /* see if the sig group head of each address group is the
          * same as an earlier one. If it is, free our head and use
@@ -3722,62 +3723,6 @@ int BuildDestinationAddressHeads(DetectEngineCtx *de_ctx, DetectAddressHead *hea
                     printf("PatternMatchPrepareGroup failed\n");
                     goto error;
                 }
-                if (sgr->sh->mpm_proto_tcp_ctx_ts != NULL) {
-                    if (de_ctx->mpm_max_patcnt < sgr->sh->mpm_proto_tcp_ctx_ts->pattern_cnt)
-                        de_ctx->mpm_max_patcnt = sgr->sh->mpm_proto_tcp_ctx_ts->pattern_cnt;
-
-                    de_ctx->mpm_tot_patcnt += sgr->sh->mpm_proto_tcp_ctx_ts->pattern_cnt;
-                }
-                if (sgr->sh->mpm_proto_tcp_ctx_tc != NULL) {
-                    if (de_ctx->mpm_max_patcnt < sgr->sh->mpm_proto_tcp_ctx_tc->pattern_cnt)
-                        de_ctx->mpm_max_patcnt = sgr->sh->mpm_proto_tcp_ctx_tc->pattern_cnt;
-
-                    de_ctx->mpm_tot_patcnt += sgr->sh->mpm_proto_tcp_ctx_tc->pattern_cnt;
-                }
-                if (sgr->sh->mpm_proto_udp_ctx_ts != NULL) {
-                    if (de_ctx->mpm_max_patcnt < sgr->sh->mpm_proto_udp_ctx_ts->pattern_cnt)
-                        de_ctx->mpm_max_patcnt = sgr->sh->mpm_proto_udp_ctx_ts->pattern_cnt;
-
-                    de_ctx->mpm_tot_patcnt += sgr->sh->mpm_proto_udp_ctx_ts->pattern_cnt;
-                }
-                if (sgr->sh->mpm_proto_udp_ctx_tc != NULL) {
-                    if (de_ctx->mpm_max_patcnt < sgr->sh->mpm_proto_udp_ctx_tc->pattern_cnt)
-                        de_ctx->mpm_max_patcnt = sgr->sh->mpm_proto_udp_ctx_tc->pattern_cnt;
-
-                    de_ctx->mpm_tot_patcnt += sgr->sh->mpm_proto_udp_ctx_tc->pattern_cnt;
-                }
-                if (sgr->sh->mpm_proto_other_ctx != NULL) {
-                    if (de_ctx->mpm_max_patcnt < sgr->sh->mpm_proto_other_ctx->pattern_cnt)
-                        de_ctx->mpm_max_patcnt = sgr->sh->mpm_proto_other_ctx->pattern_cnt;
-
-                    de_ctx->mpm_tot_patcnt += sgr->sh->mpm_proto_other_ctx->pattern_cnt;
-                }
-                if (sgr->sh->mpm_uri_ctx_ts != NULL) {
-                    if (de_ctx->mpm_uri_max_patcnt < sgr->sh->mpm_uri_ctx_ts->pattern_cnt)
-                        de_ctx->mpm_uri_max_patcnt = sgr->sh->mpm_uri_ctx_ts->pattern_cnt;
-
-                    de_ctx->mpm_uri_tot_patcnt += sgr->sh->mpm_uri_ctx_ts->pattern_cnt;
-                }
-                /* dbg */
-                if (!(sgr->sh->flags & SIG_GROUP_HEAD_MPM_COPY) && sgr->sh->mpm_proto_tcp_ctx_ts) {
-                    de_ctx->mpm_memory_size += sgr->sh->mpm_proto_tcp_ctx_ts->memory_size;
-                }
-                if (!(sgr->sh->flags & SIG_GROUP_HEAD_MPM_COPY) && sgr->sh->mpm_proto_tcp_ctx_tc) {
-                    de_ctx->mpm_memory_size += sgr->sh->mpm_proto_tcp_ctx_tc->memory_size;
-                }
-                if (!(sgr->sh->flags & SIG_GROUP_HEAD_MPM_COPY) && sgr->sh->mpm_proto_udp_ctx_ts) {
-                    de_ctx->mpm_memory_size += sgr->sh->mpm_proto_udp_ctx_ts->memory_size;
-                }
-                if (!(sgr->sh->flags & SIG_GROUP_HEAD_MPM_COPY) && sgr->sh->mpm_proto_udp_ctx_tc) {
-                    de_ctx->mpm_memory_size += sgr->sh->mpm_proto_udp_ctx_tc->memory_size;
-                }
-                if (!(sgr->sh->flags & SIG_GROUP_HEAD_MPM_COPY) && sgr->sh->mpm_proto_other_ctx) {
-                    de_ctx->mpm_memory_size += sgr->sh->mpm_proto_other_ctx->memory_size;
-                }
-                if (!(sgr->sh->flags & SIG_GROUP_HEAD_MPM_URI_COPY) && sgr->sh->mpm_uri_ctx_ts) {
-                    de_ctx->mpm_memory_size += sgr->sh->mpm_uri_ctx_ts->memory_size;
-                }
-
                 SigGroupHeadHashAdd(de_ctx, sgr->sh);
                 SigGroupHeadStore(de_ctx, sgr->sh);
                 de_ctx->gh_unique++;
@@ -3871,7 +3816,8 @@ int BuildDestinationAddressHeadsWithBothPorts(DetectEngineCtx *de_ctx, DetectAdd
          * mind the limits we use. */
         int groups = (flow ? de_ctx->max_uniq_toserver_dst_groups : de_ctx->max_uniq_toclient_dst_groups);
 
-        CreateGroupedAddrList(de_ctx, tmp_gr_list, family, src_gr->dst_gh, groups, CreateGroupedAddrListCmpMpmMaxlen, max_idx);
+        CreateGroupedAddrList(de_ctx, tmp_gr_list, family, src_gr->dst_gh, groups,
+                CreateGroupedAddrListCmpMpmMinlen, max_idx);
 
         /* add the ports to the dst address groups and the sigs
          * to the ports */
@@ -3922,7 +3868,8 @@ int BuildDestinationAddressHeadsWithBothPorts(DetectEngineCtx *de_ctx, DetectAdd
 
                 int spgroups = (flow ? de_ctx->max_uniq_toserver_sp_groups : de_ctx->max_uniq_toclient_sp_groups);
 
-                CreateGroupedPortList(de_ctx, de_ctx->sport_hash_table, &dst_gr->port, spgroups, CreateGroupedPortListCmpMpmMaxlen, max_idx);
+                CreateGroupedPortList(de_ctx, de_ctx->sport_hash_table, &dst_gr->port, spgroups,
+                        CreateGroupedPortListCmpMpmMinlen, max_idx);
 
                 SCLogDebug("adding sgh %p to the hash", dst_gr->sh);
                 SigGroupHeadHashAdd(de_ctx, dst_gr->sh);
@@ -3978,7 +3925,7 @@ int BuildDestinationAddressHeadsWithBothPorts(DetectEngineCtx *de_ctx, DetectAdd
 
                         CreateGroupedPortList(de_ctx, de_ctx->dport_hash_table,
                             &sp->dst_ph, dpgroups,
-                            CreateGroupedPortListCmpMpmMaxlen, max_idx);
+                            CreateGroupedPortListCmpMpmMinlen, max_idx);
 
                         SigGroupHeadSPortHashAdd(de_ctx, sp->sh);
 
@@ -4010,62 +3957,6 @@ int BuildDestinationAddressHeadsWithBothPorts(DetectEngineCtx *de_ctx, DetectAdd
                                     printf("PatternMatchPrepareGroup failed\n");
                                     goto error;
                                 }
-                                if (dp->sh->mpm_proto_tcp_ctx_ts != NULL) {
-                                    if (de_ctx->mpm_max_patcnt < dp->sh->mpm_proto_tcp_ctx_ts->pattern_cnt)
-                                        de_ctx->mpm_max_patcnt = dp->sh->mpm_proto_tcp_ctx_ts->pattern_cnt;
-
-                                    de_ctx->mpm_tot_patcnt += dp->sh->mpm_proto_tcp_ctx_ts->pattern_cnt;
-                                }
-                                if (dp->sh->mpm_proto_tcp_ctx_tc != NULL) {
-                                    if (de_ctx->mpm_max_patcnt < dp->sh->mpm_proto_tcp_ctx_tc->pattern_cnt)
-                                        de_ctx->mpm_max_patcnt = dp->sh->mpm_proto_tcp_ctx_tc->pattern_cnt;
-
-                                    de_ctx->mpm_tot_patcnt += dp->sh->mpm_proto_tcp_ctx_tc->pattern_cnt;
-                                }
-                                if (dp->sh->mpm_proto_udp_ctx_ts != NULL) {
-                                    if (de_ctx->mpm_max_patcnt < dp->sh->mpm_proto_udp_ctx_ts->pattern_cnt)
-                                        de_ctx->mpm_max_patcnt = dp->sh->mpm_proto_udp_ctx_ts->pattern_cnt;
-
-                                    de_ctx->mpm_tot_patcnt += dp->sh->mpm_proto_udp_ctx_ts->pattern_cnt;
-                                }
-                                if (dp->sh->mpm_proto_udp_ctx_tc != NULL) {
-                                    if (de_ctx->mpm_max_patcnt < dp->sh->mpm_proto_udp_ctx_tc->pattern_cnt)
-                                        de_ctx->mpm_max_patcnt = dp->sh->mpm_proto_udp_ctx_tc->pattern_cnt;
-
-                                    de_ctx->mpm_tot_patcnt += dp->sh->mpm_proto_udp_ctx_tc->pattern_cnt;
-                                }
-                                if (dp->sh->mpm_proto_other_ctx != NULL) {
-                                    if (de_ctx->mpm_max_patcnt < dp->sh->mpm_proto_other_ctx->pattern_cnt)
-                                        de_ctx->mpm_max_patcnt = dp->sh->mpm_proto_other_ctx->pattern_cnt;
-
-                                    de_ctx->mpm_tot_patcnt += dp->sh->mpm_proto_other_ctx->pattern_cnt;
-                                }
-                                if (dp->sh->mpm_uri_ctx_ts != NULL) {
-                                    if (de_ctx->mpm_uri_max_patcnt < dp->sh->mpm_uri_ctx_ts->pattern_cnt)
-                                        de_ctx->mpm_uri_max_patcnt = dp->sh->mpm_uri_ctx_ts->pattern_cnt;
-
-                                    de_ctx->mpm_uri_tot_patcnt += dp->sh->mpm_uri_ctx_ts->pattern_cnt;
-                                }
-                                /* dbg */
-                                if (!(dp->sh->flags & SIG_GROUP_HEAD_MPM_COPY) && dp->sh->mpm_proto_tcp_ctx_ts) {
-                                    de_ctx->mpm_memory_size += dp->sh->mpm_proto_tcp_ctx_ts->memory_size;
-                                }
-                                if (!(dp->sh->flags & SIG_GROUP_HEAD_MPM_COPY) && dp->sh->mpm_proto_tcp_ctx_tc) {
-                                    de_ctx->mpm_memory_size += dp->sh->mpm_proto_tcp_ctx_tc->memory_size;
-                                }
-                                if (!(dp->sh->flags & SIG_GROUP_HEAD_MPM_COPY) && dp->sh->mpm_proto_udp_ctx_ts) {
-                                    de_ctx->mpm_memory_size += dp->sh->mpm_proto_udp_ctx_ts->memory_size;
-                                }
-                                if (!(dp->sh->flags & SIG_GROUP_HEAD_MPM_COPY) && dp->sh->mpm_proto_udp_ctx_tc) {
-                                    de_ctx->mpm_memory_size += dp->sh->mpm_proto_udp_ctx_tc->memory_size;
-                                }
-                                if (!(dp->sh->flags & SIG_GROUP_HEAD_MPM_COPY) && dp->sh->mpm_proto_other_ctx) {
-                                    de_ctx->mpm_memory_size += dp->sh->mpm_proto_other_ctx->memory_size;
-                                }
-                                if (!(dp->sh->flags & SIG_GROUP_HEAD_MPM_URI_COPY) && dp->sh->mpm_uri_ctx_ts) {
-                                    de_ctx->mpm_memory_size += dp->sh->mpm_uri_ctx_ts->memory_size;
-                                }
-
                                 SigGroupHeadDPortHashAdd(de_ctx, dp->sh);
                                 SigGroupHeadStore(de_ctx, dp->sh);
                                 de_ctx->gh_unique++;
@@ -4250,20 +4141,8 @@ int SigAddressPrepareStage3(DetectEngineCtx *de_ctx)
     DetectPortSpHashFree(de_ctx);
 
     if (!(de_ctx->flags & DE_QUIET)) {
-        SCLogDebug("MPM memory %" PRIuMAX " (dynamic %" PRIu32 ", ctxs %" PRIuMAX ", avg per ctx %" PRIu32 ")",
-            de_ctx->mpm_memory_size + ((de_ctx->mpm_unique + de_ctx->mpm_uri_unique) * (uintmax_t)sizeof(MpmCtx)),
-            de_ctx->mpm_memory_size, ((de_ctx->mpm_unique + de_ctx->mpm_uri_unique) * (uintmax_t)sizeof(MpmCtx)),
-            de_ctx->mpm_unique ? de_ctx->mpm_memory_size / de_ctx->mpm_unique: 0);
-
         SCLogDebug("max sig id %" PRIu32 ", array size %" PRIu32 "", DetectEngineGetMaxSigId(de_ctx), DetectEngineGetMaxSigId(de_ctx) / 8 + 1);
         SCLogDebug("signature group heads: unique %" PRIu32 ", copies %" PRIu32 ".", de_ctx->gh_unique, de_ctx->gh_reuse);
-        SCLogDebug("MPM instances: %" PRIu32 " unique, copies %" PRIu32 " (none %" PRIu32 ").",
-                de_ctx->mpm_unique, de_ctx->mpm_reuse, de_ctx->mpm_none);
-        SCLogDebug("MPM (URI) instances: %" PRIu32 " unique, copies %" PRIu32 " (none %" PRIu32 ").",
-                de_ctx->mpm_uri_unique, de_ctx->mpm_uri_reuse, de_ctx->mpm_uri_none);
-        SCLogDebug("MPM max patcnt %" PRIu32 ", avg %" PRIu32 "", de_ctx->mpm_max_patcnt, de_ctx->mpm_unique?de_ctx->mpm_tot_patcnt/de_ctx->mpm_unique:0);
-        if (de_ctx->mpm_uri_tot_patcnt && de_ctx->mpm_uri_unique)
-            SCLogDebug("MPM (URI) max patcnt %" PRIu32 ", avg %" PRIu32 " (%" PRIu32 "/%" PRIu32 ")", de_ctx->mpm_uri_max_patcnt, de_ctx->mpm_uri_tot_patcnt/de_ctx->mpm_uri_unique, de_ctx->mpm_uri_tot_patcnt, de_ctx->mpm_uri_unique);
         SCLogDebug("port maxgroups: %" PRIu32 ", avg %" PRIu32 ", tot %" PRIu32 "", g_groupportlist_maxgroups, g_groupportlist_groupscnt ? g_groupportlist_totgroups/g_groupportlist_groupscnt : 0, g_groupportlist_totgroups);
 
         SCLogInfo("building signature grouping structure, stage 3: building destination address lists... complete");
@@ -4376,6 +4255,9 @@ int SigAddressPrepareStage4(DetectEngineCtx *de_ctx)
         SCLogDebug("filestore count %u", sgh->filestore_cnt);
 
         SigGroupHeadBuildNonMpmArray(de_ctx, sgh);
+
+        sgh->mpm_uricontent_minlen = SigGroupHeadGetMinMpmSize(de_ctx, sgh, DETECT_SM_LIST_UMATCH);
+        SCLogDebug("http_uri content min mpm len: %u", sgh->mpm_uricontent_minlen);
     }
 
     if (de_ctx->decoder_event_sgh != NULL) {
@@ -4471,7 +4353,7 @@ int SigAddressPrepareStage5(DetectEngineCtx *de_ctx)
                         DetectPort *dp = sp->dst_ph;
                         for ( ; dp != NULL; dp = dp->next) {
                             printf("   4 Dst port(range): "); DetectPortPrint(dp);
-                            printf(" (sigs %" PRIu32 ", sgh %p, maxlen %" PRIu32 ")", dp->sh->sig_cnt, dp->sh, dp->sh->mpm_content_maxlen);
+                            printf(" (sigs %" PRIu32 ", sgh %p, minlen %" PRIu32 ")", dp->sh->sig_cnt, dp->sh, dp->sh->mpm_content_minlen);
 #ifdef PRINTSIGS
                             printf(" - ");
                             for (u = 0; u < dp->sh->sig_cnt; u++) {
@@ -9913,10 +9795,6 @@ static int SigTestSgh01 (void)
         printf("internal id != 0: ");
         goto end;
     }
-    if (de_ctx->sig_list->mpm_content_maxlen != 3) {
-        printf("de_ctx->sig_list->mpm_content_maxlen %u, expected 3: ", de_ctx->sig_list->mpm_content_maxlen);
-        goto end;
-    }
 
     de_ctx->sig_list->next = SigInit(de_ctx,"alert tcp any any -> any 81 (msg:\"2\"; content:\"two\"; content:\"abcd\"; sid:2;)");
     if (de_ctx->sig_list->next == NULL) {
@@ -9927,10 +9805,6 @@ static int SigTestSgh01 (void)
         printf("internal id != 1: ");
         goto end;
     }
-    if (de_ctx->sig_list->next->mpm_content_maxlen != 4) {
-        printf("de_ctx->sig_list->mpm_content_maxlen %u, expected 4: ", de_ctx->sig_list->next->mpm_content_maxlen);
-        goto end;
-    }
 
     de_ctx->sig_list->next->next = SigInit(de_ctx,"alert tcp any any -> any 80 (msg:\"3\"; content:\"three\"; sid:3;)");
     if (de_ctx->sig_list->next->next == NULL) {
@@ -9941,10 +9815,6 @@ static int SigTestSgh01 (void)
         printf("internal id != 2: ");
         goto end;
     }
-    if (de_ctx->sig_list->next->next->mpm_content_maxlen != 5) {
-        printf("de_ctx->sig_list->next->next->mpm_content_maxlen %u, expected 5: ", de_ctx->sig_list->next->next->mpm_content_maxlen);
-        goto end;
-    }
 
     SigGroupBuild(de_ctx);
     DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx);
@@ -9961,8 +9831,8 @@ static int SigTestSgh01 (void)
     printf("sgh->sig_cnt %u\n", sgh->sig_cnt);
     printf("sgh->sig_size %u\n", sgh->sig_size);
 #endif
-    if (sgh->mpm_content_maxlen != 3) {
-        printf("sgh->mpm_content_maxlen %u, expected 3: ", sgh->mpm_content_maxlen);
+    if (sgh->mpm_content_minlen != 3) {
+        printf("sgh->mpm_content_minlen %u, expected 3: ", sgh->mpm_content_minlen);
         goto end;
     }
 
@@ -10006,13 +9876,13 @@ static int SigTestSgh01 (void)
 
 #if 0
     printf("-\n");
-    printf("sgh2->mpm_content_maxlen %u\n", sgh2->mpm_content_maxlen);
-    printf("sgh2->mpm_uricontent_maxlen %u\n", sgh2->mpm_uricontent_maxlen);
+    printf("sgh2->mpm_content_minlen %u\n", sgh2->mpm_content_minlen);
+    printf("sgh2->mpm_uricontent_minlen %u\n", sgh2->mpm_uricontent_minlen);
     printf("sgh2->sig_cnt %u\n", sgh2->sig_cnt);
     printf("sgh2->sig_size %u\n", sgh2->sig_size);
 #endif
-    if (sgh2->mpm_content_maxlen != 4) {
-        printf("sgh2->mpm_content_maxlen %u, expected 4: ", sgh2->mpm_content_maxlen);
+    if (sgh2->mpm_content_minlen != 4) {
+        printf("sgh2->mpm_content_minlen %u, expected 4: ", sgh2->mpm_content_minlen);
         goto end;
     }
 
@@ -10084,8 +9954,8 @@ static int SigTestSgh02 (void)
         goto end;
     }
 
-    if (sgh->mpm_content_maxlen != 3) {
-        printf("sgh->mpm_content_maxlen %u, expected 3: ", sgh->mpm_content_maxlen);
+    if (sgh->mpm_content_minlen != 3) {
+        printf("sgh->mpm_content_minlen %u, expected 3: ", sgh->mpm_content_minlen);
         goto end;
     }
 
@@ -10108,8 +9978,8 @@ static int SigTestSgh02 (void)
         goto end;
     }
 #if 0
-    printf("sgh->mpm_content_maxlen %u\n", sgh->mpm_content_maxlen);
-    printf("sgh->mpm_uricontent_maxlen %u\n", sgh->mpm_uricontent_maxlen);
+    printf("sgh->mpm_content_minlen %u\n", sgh->mpm_content_minlen);
+    printf("sgh->mpm_uricontent_minlen %u\n", sgh->mpm_uricontent_minlen);
     printf("sgh->sig_cnt %u\n", sgh->sig_cnt);
     printf("sgh->sig_size %u\n", sgh->sig_size);
     printf("sgh->refcnt %u\n", sgh->refcnt);
@@ -10122,8 +9992,8 @@ static int SigTestSgh02 (void)
         goto end;
     }
 
-    if (sgh->mpm_content_maxlen != 3) {
-        printf("sgh->mpm_content_maxlen %u, expected 3: ", sgh->mpm_content_maxlen);
+    if (sgh->mpm_content_minlen != 3) {
+        printf("sgh->mpm_content_minlen %u, expected 3: ", sgh->mpm_content_minlen);
         goto end;
     }
 
@@ -10144,8 +10014,8 @@ static int SigTestSgh02 (void)
         goto end;
     }
 #if 0
-    printf("sgh->mpm_content_maxlen %u\n", sgh->mpm_content_maxlen);
-    printf("sgh->mpm_uricontent_maxlen %u\n", sgh->mpm_uricontent_maxlen);
+    printf("sgh->mpm_content_minlen %u\n", sgh->mpm_content_minlen);
+    printf("sgh->mpm_uricontent_minlen %u\n", sgh->mpm_uricontent_minlen);
     printf("sgh->sig_cnt %u\n", sgh->sig_cnt);
     printf("sgh->sig_size %u\n", sgh->sig_size);
     printf("sgh->refcnt %u\n", sgh->refcnt);
@@ -10158,8 +10028,8 @@ static int SigTestSgh02 (void)
         goto end;
     }
 
-    if (sgh->mpm_content_maxlen != 3) {
-        printf("sgh->mpm_content_maxlen %u, expected 3: ", sgh->mpm_content_maxlen);
+    if (sgh->mpm_content_minlen != 3) {
+        printf("sgh->mpm_content_minlen %u, expected 3: ", sgh->mpm_content_minlen);
         goto end;
     }
 
@@ -10173,8 +10043,8 @@ static int SigTestSgh02 (void)
         goto end;
     }
 #if 0
-    printf("sgh->mpm_content_maxlen %u\n", sgh->mpm_content_maxlen);
-    printf("sgh->mpm_uricontent_maxlen %u\n", sgh->mpm_uricontent_maxlen);
+    printf("sgh->mpm_content_minlen %u\n", sgh->mpm_content_minlen);
+    printf("sgh->mpm_uricontent_minlen %u\n", sgh->mpm_uricontent_minlen);
     printf("sgh->sig_cnt %u\n", sgh->sig_cnt);
     printf("sgh->sig_size %u\n", sgh->sig_size);
     printf("sgh->refcnt %u\n", sgh->refcnt);
@@ -10188,8 +10058,8 @@ static int SigTestSgh02 (void)
         goto end;
     }
 
-    if (sgh->mpm_content_maxlen != 3) {
-        printf("sgh->mpm_content_maxlen %u, expected 3: ", sgh->mpm_content_maxlen);
+    if (sgh->mpm_content_minlen != 3) {
+        printf("sgh->mpm_content_minlen %u, expected 3: ", sgh->mpm_content_minlen);
         goto end;
     }
 
@@ -10203,8 +10073,8 @@ static int SigTestSgh02 (void)
         goto end;
     }
 #if 0
-    printf("sgh->mpm_content_maxlen %u\n", sgh->mpm_content_maxlen);
-    printf("sgh->mpm_uricontent_maxlen %u\n", sgh->mpm_uricontent_maxlen);
+    printf("sgh->mpm_content_minlen %u\n", sgh->mpm_content_minlen);
+    printf("sgh->mpm_uricontent_minlen %u\n", sgh->mpm_uricontent_minlen);
     printf("sgh->sig_cnt %u\n", sgh->sig_cnt);
     printf("sgh->sig_size %u\n", sgh->sig_size);
     printf("sgh->refcnt %u\n", sgh->refcnt);
@@ -10281,14 +10151,14 @@ static int SigTestSgh03 (void)
     }
 #if 0
     printf("-\n");
-    printf("sgh->mpm_content_maxlen %u\n", sgh->mpm_content_maxlen);
-    printf("sgh->mpm_uricontent_maxlen %u\n", sgh->mpm_uricontent_maxlen);
+    printf("sgh->mpm_content_minlen %u\n", sgh->mpm_content_minlen);
+    printf("sgh->mpm_uricontent_minlen %u\n", sgh->mpm_uricontent_minlen);
     printf("sgh->sig_cnt %u\n", sgh->sig_cnt);
     printf("sgh->sig_size %u\n", sgh->sig_size);
     printf("sgh->refcnt %u\n", sgh->refcnt);
 #endif
-    if (sgh->mpm_content_maxlen != 3) {
-        printf("sgh->mpm_content_maxlen %u, expected 3: ", sgh->mpm_content_maxlen);
+    if (sgh->mpm_content_minlen != 3) {
+        printf("sgh->mpm_content_minlen %u, expected 3: ", sgh->mpm_content_minlen);
         goto end;
     }
 
@@ -10321,8 +10191,8 @@ static int SigTestSgh03 (void)
 #if 0
     printf("-\n");
     printf("sgh %p\n", sgh);
-    printf("sgh->mpm_content_maxlen %u\n", sgh->mpm_content_maxlen);
-    printf("sgh->mpm_uricontent_maxlen %u\n", sgh->mpm_uricontent_maxlen);
+    printf("sgh->mpm_content_minlen %u\n", sgh->mpm_content_minlen);
+    printf("sgh->mpm_uricontent_minlen %u\n", sgh->mpm_uricontent_minlen);
     printf("sgh->sig_cnt %u\n", sgh->sig_cnt);
     printf("sgh->sig_size %u\n", sgh->sig_size);
     printf("sgh->refcnt %u\n", sgh->refcnt);
@@ -10344,8 +10214,8 @@ static int SigTestSgh03 (void)
         goto end;
     }
 
-    if (sgh->mpm_content_maxlen != 3) {
-        printf("sgh->mpm_content_maxlen %u, expected 3 (%x): ", sgh->mpm_content_maxlen, p->dst.addr_data32[0]);
+    if (sgh->mpm_content_minlen != 3) {
+        printf("sgh->mpm_content_minlen %u, expected 3 (%x): ", sgh->mpm_content_minlen, p->dst.addr_data32[0]);
         goto end;
     }
 
@@ -10359,14 +10229,14 @@ static int SigTestSgh03 (void)
     }
 #if 0
     printf("-\n");
-    printf("sgh->mpm_content_maxlen %u\n", sgh->mpm_content_maxlen);
-    printf("sgh->mpm_uricontent_maxlen %u\n", sgh->mpm_uricontent_maxlen);
+    printf("sgh->mpm_content_minlen %u\n", sgh->mpm_content_minlen);
+    printf("sgh->mpm_uricontent_minlen %u\n", sgh->mpm_uricontent_minlen);
     printf("sgh->sig_cnt %u\n", sgh->sig_cnt);
     printf("sgh->sig_size %u\n", sgh->sig_size);
     printf("sgh->refcnt %u\n", sgh->refcnt);
 #endif
-    if (sgh->mpm_content_maxlen != 3) {
-        printf("sgh->mpm_content_maxlen %u, expected 3: ", sgh->mpm_content_maxlen);
+    if (sgh->mpm_content_minlen != 3) {
+        printf("sgh->mpm_content_minlen %u, expected 3: ", sgh->mpm_content_minlen);
         goto end;
     }
 
@@ -10452,8 +10322,8 @@ static int SigTestSgh04 (void)
         goto end;
     }
 
-    if (sgh->mpm_content_maxlen != 3) {
-        printf("sgh->mpm_content_maxlen %u, expected 3: ", sgh->mpm_content_maxlen);
+    if (sgh->mpm_content_minlen != 3) {
+        printf("sgh->mpm_content_minlen %u, expected 3: ", sgh->mpm_content_minlen);
         goto end;
     }
 
@@ -10476,8 +10346,8 @@ static int SigTestSgh04 (void)
         goto end;
     }
 #if 0
-    printf("sgh->mpm_content_maxlen %u\n", sgh->mpm_content_maxlen);
-    printf("sgh->mpm_uricontent_maxlen %u\n", sgh->mpm_uricontent_maxlen);
+    printf("sgh->mpm_content_minlen %u\n", sgh->mpm_content_minlen);
+    printf("sgh->mpm_uricontent_minlen %u\n", sgh->mpm_uricontent_minlen);
     printf("sgh->sig_cnt %u\n", sgh->sig_cnt);
     printf("sgh->sig_size %u\n", sgh->sig_size);
     printf("sgh->refcnt %u\n", sgh->refcnt);
@@ -10490,8 +10360,8 @@ static int SigTestSgh04 (void)
         goto end;
     }
 
-    if (sgh->mpm_content_maxlen != 3) {
-        printf("sgh->mpm_content_maxlen %u, expected 3: ", sgh->mpm_content_maxlen);
+    if (sgh->mpm_content_minlen != 3) {
+        printf("sgh->mpm_content_minlen %u, expected 3: ", sgh->mpm_content_minlen);
         goto end;
     }
 
@@ -10512,8 +10382,8 @@ static int SigTestSgh04 (void)
         goto end;
     }
 #if 0
-    printf("sgh->mpm_content_maxlen %u\n", sgh->mpm_content_maxlen);
-    printf("sgh->mpm_uricontent_maxlen %u\n", sgh->mpm_uricontent_maxlen);
+    printf("sgh->mpm_content_minlen %u\n", sgh->mpm_content_minlen);
+    printf("sgh->mpm_uricontent_minlen %u\n", sgh->mpm_uricontent_minlen);
     printf("sgh->sig_cnt %u\n", sgh->sig_cnt);
     printf("sgh->sig_size %u\n", sgh->sig_size);
     printf("sgh->refcnt %u\n", sgh->refcnt);
@@ -10526,8 +10396,8 @@ static int SigTestSgh04 (void)
         goto end;
     }
 
-    if (sgh->mpm_content_maxlen != 3) {
-        printf("sgh->mpm_content_maxlen %u, expected 3: ", sgh->mpm_content_maxlen);
+    if (sgh->mpm_content_minlen != 3) {
+        printf("sgh->mpm_content_minlen %u, expected 3: ", sgh->mpm_content_minlen);
         goto end;
     }
 
@@ -10541,8 +10411,8 @@ static int SigTestSgh04 (void)
         goto end;
     }
 #if 0
-    printf("sgh->mpm_content_maxlen %u\n", sgh->mpm_content_maxlen);
-    printf("sgh->mpm_uricontent_maxlen %u\n", sgh->mpm_uricontent_maxlen);
+    printf("sgh->mpm_content_minlen %u\n", sgh->mpm_content_minlen);
+    printf("sgh->mpm_uricontent_minlen %u\n", sgh->mpm_uricontent_minlen);
     printf("sgh->sig_cnt %u\n", sgh->sig_cnt);
     printf("sgh->sig_size %u\n", sgh->sig_size);
     printf("sgh->refcnt %u\n", sgh->refcnt);
@@ -10556,14 +10426,14 @@ static int SigTestSgh04 (void)
     }
 #if 0
     printf("-\n");
-    printf("sgh->mpm_content_maxlen %u\n", sgh->mpm_content_maxlen);
-    printf("sgh->mpm_uricontent_maxlen %u\n", sgh->mpm_uricontent_maxlen);
+    printf("sgh->mpm_content_minlen %u\n", sgh->mpm_content_minlen);
+    printf("sgh->mpm_uricontent_minlen %u\n", sgh->mpm_uricontent_minlen);
     printf("sgh->sig_cnt %u\n", sgh->sig_cnt);
     printf("sgh->sig_size %u\n", sgh->sig_size);
     printf("sgh->refcnt %u\n", sgh->refcnt);
 #endif
-    if (sgh->mpm_content_maxlen != 3) {
-        printf("sgh->mpm_content_maxlen %u, expected 3: ", sgh->mpm_content_maxlen);
+    if (sgh->mpm_content_minlen != 3) {
+        printf("sgh->mpm_content_minlen %u, expected 3: ", sgh->mpm_content_minlen);
         goto end;
     }
 
diff --git a/framework/src/suricata/src/detect.h b/framework/src/suricata/src/detect.h
index 30adc9c4..8d131b92 100644
--- a/framework/src/suricata/src/detect.h
+++ b/framework/src/suricata/src/detect.h
@@ -449,10 +449,8 @@ typedef struct Signature_ {
     SigMatch *dsize_sm;
     /* the fast pattern added from this signature */
     SigMatch *mpm_sm;
-    /* helper for init phase */
-    uint16_t mpm_content_maxlen;
-    uint16_t mpm_uricontent_maxlen;
 
+    /* SigMatch list used for adding content and friends. E.g. file_data; */
     int list;
 
     /* Be careful, this pointer is only valid while parsing the sig,
@@ -595,13 +593,8 @@ typedef struct DetectEngineCtx_ {
     /* main sigs */
     DetectEngineLookupFlow flow_gh[FLOW_STATES];
 
-    uint32_t mpm_unique, mpm_reuse, mpm_none,
-        mpm_uri_unique, mpm_uri_reuse, mpm_uri_none;
     uint32_t gh_unique, gh_reuse;
 
-    uint32_t mpm_max_patcnt, mpm_min_patcnt, mpm_tot_patcnt,
-        mpm_uri_max_patcnt, mpm_uri_min_patcnt, mpm_uri_tot_patcnt;
-
     /* init phase vars */
     HashListTable *sgh_hash_table;
 
@@ -622,9 +615,6 @@ typedef struct DetectEngineCtx_ {
     /* hash table used to cull out duplicate sigs */
     HashListTable *dup_sig_hash_table;
 
-    /* memory counters */
-    uint32_t mpm_memory_size;
-
     DetectEngineIPOnlyCtx io_ctx;
     ThresholdCtx ths_ctx;
 
@@ -994,7 +984,8 @@ typedef struct SigGroupHead_ {
     /* number of sigs in this head */
     SigIntId sig_cnt;
 
-    uint16_t mpm_content_maxlen;
+    /* track min pattern length for content. Used in grouping */
+    uint16_t mpm_content_minlen;
 
     /** array of masks, used to check multiple masks against
      *  a packet using SIMD. */
@@ -1034,7 +1025,7 @@ typedef struct SigGroupHead_ {
     MpmCtx *mpm_hsmd_ctx_tc;
     MpmCtx *mpm_hscd_ctx_tc;
 
-    uint16_t mpm_uricontent_maxlen;
+    uint16_t mpm_uricontent_minlen; /**< len of shortest mpm pattern in sgh */
 
     /** the number of signatures in this sgh that have the filestore keyword
      *  set. */
diff --git a/framework/src/suricata/src/flow-hash.c b/framework/src/suricata/src/flow-hash.c
index 7a151199..9ddb3713 100644
--- a/framework/src/suricata/src/flow-hash.c
+++ b/framework/src/suricata/src/flow-hash.c
@@ -35,6 +35,7 @@
 #include "flow-util.h"
 #include "flow-private.h"
 #include "flow-manager.h"
+#include "flow-storage.h"
 #include "app-layer-parser.h"
 
 #include "util-time.h"
@@ -472,7 +473,7 @@ static Flow *FlowGetNew(ThreadVars *tv, DecodeThreadVars *dtv, const Packet *p)
     f = FlowDequeue(&flow_spare_q);
     if (f == NULL) {
         /* If we reached the max memcap, we get a used flow */
-        if (!(FLOW_CHECK_MEMCAP(sizeof(Flow)))) {
+        if (!(FLOW_CHECK_MEMCAP(sizeof(Flow) + FlowStorageSize()))) {
             /* declare state of emergency */
             if (!(SC_ATOMIC_GET(flow_flags) & FLOW_EMERGENCY)) {
                 SC_ATOMIC_OR(flow_flags, FLOW_EMERGENCY);
@@ -485,6 +486,11 @@ static Flow *FlowGetNew(ThreadVars *tv, DecodeThreadVars *dtv, const Packet *p)
 
             f = FlowGetUsedFlow(tv, dtv);
             if (f == NULL) {
+                /* max memcap reached, so increments the counter */
+                if (tv != NULL && dtv != NULL) {
+                    StatsIncr(tv, dtv->counter_flow_memcap);
+                }
+
                 /* very rare, but we can fail. Just giving up */
                 return NULL;
             }
@@ -494,6 +500,9 @@ static Flow *FlowGetNew(ThreadVars *tv, DecodeThreadVars *dtv, const Packet *p)
             /* now see if we can alloc a new flow */
             f = FlowAlloc();
             if (f == NULL) {
+                if (tv != NULL && dtv != NULL) {
+                    StatsIncr(tv, dtv->counter_flow_memcap);
+                }
                 return NULL;
             }
 
diff --git a/framework/src/suricata/src/flow.c b/framework/src/suricata/src/flow.c
index 42099c3a..abd9e7e5 100644
--- a/framework/src/suricata/src/flow.c
+++ b/framework/src/suricata/src/flow.c
@@ -443,7 +443,7 @@ void FlowInitConfig(char quiet)
 
     /* pre allocate flows */
     for (i = 0; i < flow_config.prealloc; i++) {
-        if (!(FLOW_CHECK_MEMCAP(sizeof(Flow)))) {
+        if (!(FLOW_CHECK_MEMCAP(sizeof(Flow) + FlowStorageSize()))) {
             SCLogError(SC_ERR_FLOW_INIT, "preallocating flows failed: "
                     "max flow memcap reached. Memcap %"PRIu64", "
                     "Memuse %"PRIu64".", flow_config.memcap,
@@ -462,7 +462,7 @@ void FlowInitConfig(char quiet)
 
     if (quiet == FALSE) {
         SCLogInfo("preallocated %" PRIu32 " flows of size %" PRIuMAX "",
-                flow_spare_q.len, (uintmax_t)sizeof(Flow));
+                flow_spare_q.len, (uintmax_t)(sizeof(Flow) + + FlowStorageSize()));
         SCLogInfo("flow memory usage: %llu bytes, maximum: %"PRIu64,
                 SC_ATOMIC_GET(flow_memuse), flow_config.memcap);
     }
diff --git a/framework/src/suricata/src/host-storage.c b/framework/src/suricata/src/host-storage.c
index 6748089c..fe157692 100644
--- a/framework/src/suricata/src/host-storage.c
+++ b/framework/src/suricata/src/host-storage.c
@@ -32,16 +32,66 @@ unsigned int HostStorageSize(void)
     return StorageGetSize(STORAGE_HOST);
 }
 
-void *HostGetStorageById(Host *h, int id)
-{
-    return StorageGetById((Storage *)((void *)h + sizeof(Host)), STORAGE_HOST, id);
+/** \defgroup hoststorage Host storage API
+ *
+ * The Host storage API is a per-host storage. It is a mean to extend
+ * the Host structure with arbitrary data.
+ *
+ * You have first to register the storage via HostStorageRegister() during
+ * the init of your module. Then you can attach data via HostSetStorageById()
+ * and access them via HostGetStorageById().
+ * @{
+ */
+
+/**
+ * \brief Register a Host storage
+ *
+ * \param name the name of the storage
+ * \param size integer coding the size of the stored value (sizeof(void *) is best choice here)
+ * \param Alloc allocation function for the storage (can be null)
+ * \param Free free function for the new storage
+ *
+ * \retval The ID of the newly register storage that will be used to access data
+ *
+ * It has to be called once during the init of the sub system
+ */
+
+int HostStorageRegister(const char *name, const unsigned int size, void *(*Alloc)(unsigned int), void (*Free)(void *)) {
+    return StorageRegister(STORAGE_HOST, name, size, Alloc, Free);
 }
 
+/**
+ * \brief Store a pointer in a given Host storage
+ *
+ * \param h a pointer to the Host
+ * \param id the id of the storage (return of HostStorageRegister() call)
+ * \param ptr pointer to the data to store
+ */
+
 int HostSetStorageById(Host *h, int id, void *ptr)
 {
     return StorageSetById((Storage *)((void *)h + sizeof(Host)), STORAGE_HOST, id, ptr);
 }
 
+/**
+ * \brief Get a value from a given Host storage
+ *
+ * \param h a pointer to the Host
+ * \param id the id of the storage (return of HostStorageRegister() call)
+ *
+ */
+
+void *HostGetStorageById(Host *h, int id)
+{
+    return StorageGetById((Storage *)((void *)h + sizeof(Host)), STORAGE_HOST, id);
+}
+
+/**
+ * @}
+ */
+
+/* Start of "private" function */
+
 void *HostAllocStorageById(Host *h, int id)
 {
     return StorageAllocByIdPrealloc((Storage *)((void *)h + sizeof(Host)), STORAGE_HOST, id);
@@ -58,9 +108,6 @@ void HostFreeStorage(Host *h)
         StorageFreeAll((Storage *)((void *)h + sizeof(Host)), STORAGE_HOST);
 }
 
-int HostStorageRegister(const char *name, const unsigned int size, void *(*Alloc)(unsigned int), void (*Free)(void *)) {
-    return StorageRegister(STORAGE_HOST, name, size, Alloc, Free);
-}
 
 #ifdef UNITTESTS
 
diff --git a/framework/src/suricata/src/host.c b/framework/src/suricata/src/host.c
index 7c3c5841..a28639cc 100644
--- a/framework/src/suricata/src/host.c
+++ b/framework/src/suricata/src/host.c
@@ -48,6 +48,10 @@ static Host *HostGetUsedHost(void);
 /** queue with spare hosts */
 static HostQueue host_spare_q;
 
+/** size of the host object. Maybe updated in HostInitConfig to include
+ *  the storage APIs additions. */
+static uint16_t g_host_size = sizeof(Host);
+
 uint32_t HostSpareQueueGetSize(void)
 {
     return HostQueueLen(&host_spare_q);
@@ -61,19 +65,16 @@ void HostMoveToSpare(Host *h)
 
 Host *HostAlloc(void)
 {
-    size_t size = sizeof(Host) + HostStorageSize();
-
-    if (!(HOST_CHECK_MEMCAP(size))) {
+    if (!(HOST_CHECK_MEMCAP(g_host_size))) {
         return NULL;
     }
+    (void) SC_ATOMIC_ADD(host_memuse, g_host_size);
 
-    (void) SC_ATOMIC_ADD(host_memuse, size);
-
-    Host *h = SCMalloc(size);
+    Host *h = SCMalloc(g_host_size);
     if (unlikely(h == NULL))
         goto error;
 
-    memset(h, 0x00, size);
+    memset(h, 0x00, g_host_size);
 
     SCMutexInit(&h->m, NULL);
     SC_ATOMIC_INIT(h->use_cnt);
@@ -91,7 +92,7 @@ void HostFree(Host *h)
         SC_ATOMIC_DESTROY(h->use_cnt);
         SCMutexDestroy(&h->m);
         SCFree(h);
-        (void) SC_ATOMIC_SUB(host_memuse, (sizeof(Host) + HostStorageSize()));
+        (void) SC_ATOMIC_SUB(host_memuse, g_host_size);
     }
 }
 
@@ -130,6 +131,8 @@ void HostClearMemory(Host *h)
 void HostInitConfig(char quiet)
 {
     SCLogDebug("initializing host engine...");
+    if (HostStorageSize() > 0)
+        g_host_size = sizeof(Host) + HostStorageSize();
 
     memset(&host_config,  0, sizeof(host_config));
     //SC_ATOMIC_INIT(flow_flags);
@@ -214,11 +217,11 @@ void HostInitConfig(char quiet)
 
     /* pre allocate hosts */
     for (i = 0; i < host_config.prealloc; i++) {
-        if (!(HOST_CHECK_MEMCAP(sizeof(Host)))) {
+        if (!(HOST_CHECK_MEMCAP(g_host_size))) {
             SCLogError(SC_ERR_HOST_INIT, "preallocating hosts failed: "
                     "max host memcap reached. Memcap %"PRIu64", "
                     "Memuse %"PRIu64".", host_config.memcap,
-                    ((uint64_t)SC_ATOMIC_GET(host_memuse) + (uint64_t)sizeof(Host)));
+                    ((uint64_t)SC_ATOMIC_GET(host_memuse) + g_host_size));
             exit(EXIT_FAILURE);
         }
 
@@ -231,8 +234,8 @@ void HostInitConfig(char quiet)
     }
 
     if (quiet == FALSE) {
-        SCLogInfo("preallocated %" PRIu32 " hosts of size %" PRIuMAX "",
-                host_spare_q.len, (uintmax_t)sizeof(Host));
+        SCLogInfo("preallocated %" PRIu32 " hosts of size %" PRIu16 "",
+                host_spare_q.len, g_host_size);
         SCLogInfo("host memory usage: %llu bytes, maximum: %"PRIu64,
                 SC_ATOMIC_GET(host_memuse), host_config.memcap);
     }
@@ -386,7 +389,7 @@ static Host *HostGetNew(Address *a)
     h = HostDequeue(&host_spare_q);
     if (h == NULL) {
         /* If we reached the max memcap, we get a used host */
-        if (!(HOST_CHECK_MEMCAP(sizeof(Host)))) {
+        if (!(HOST_CHECK_MEMCAP(g_host_size))) {
             /* declare state of emergency */
             //if (!(SC_ATOMIC_GET(host_flags) & HOST_EMERGENCY)) {
             //    SC_ATOMIC_OR(host_flags, HOST_EMERGENCY);
diff --git a/framework/src/suricata/src/ippair.c b/framework/src/suricata/src/ippair.c
index 780ee6ce..79ecb76c 100644
--- a/framework/src/suricata/src/ippair.c
+++ b/framework/src/suricata/src/ippair.c
@@ -47,6 +47,10 @@ static IPPair *IPPairGetUsedIPPair(void);
 /** queue with spare ippairs */
 static IPPairQueue ippair_spare_q;
 
+/** size of the ippair object. Maybe updated in IPPairInitConfig to include
+ *  the storage APIs additions. */
+static uint16_t g_ippair_size = sizeof(IPPair);
+
 uint32_t IPPairSpareQueueGetSize(void)
 {
     return IPPairQueueLen(&ippair_spare_q);
@@ -60,19 +64,17 @@ void IPPairMoveToSpare(IPPair *h)
 
 IPPair *IPPairAlloc(void)
 {
-    size_t size = sizeof(IPPair) + IPPairStorageSize();
-
-    if (!(IPPAIR_CHECK_MEMCAP(size))) {
+    if (!(IPPAIR_CHECK_MEMCAP(g_ippair_size))) {
         return NULL;
     }
 
-    (void) SC_ATOMIC_ADD(ippair_memuse, size);
+    (void) SC_ATOMIC_ADD(ippair_memuse, g_ippair_size);
 
-    IPPair *h = SCMalloc(size);
+    IPPair *h = SCMalloc(g_ippair_size);
     if (unlikely(h == NULL))
         goto error;
 
-    memset(h, 0x00, size);
+    memset(h, 0x00, g_ippair_size);
 
     SCMutexInit(&h->m, NULL);
     SC_ATOMIC_INIT(h->use_cnt);
@@ -90,7 +92,7 @@ void IPPairFree(IPPair *h)
         SC_ATOMIC_DESTROY(h->use_cnt);
         SCMutexDestroy(&h->m);
         SCFree(h);
-        (void) SC_ATOMIC_SUB(ippair_memuse, (sizeof(IPPair) + IPPairStorageSize()));
+        (void) SC_ATOMIC_SUB(ippair_memuse, g_ippair_size);
     }
 }
 
@@ -125,6 +127,8 @@ void IPPairClearMemory(IPPair *h)
 void IPPairInitConfig(char quiet)
 {
     SCLogDebug("initializing ippair engine...");
+    if (IPPairStorageSize() > 0)
+        g_ippair_size = sizeof(IPPair) + IPPairStorageSize();
 
     memset(&ippair_config,  0, sizeof(ippair_config));
     //SC_ATOMIC_INIT(flow_flags);
@@ -209,11 +213,11 @@ void IPPairInitConfig(char quiet)
 
     /* pre allocate ippairs */
     for (i = 0; i < ippair_config.prealloc; i++) {
-        if (!(IPPAIR_CHECK_MEMCAP(sizeof(IPPair)))) {
+        if (!(IPPAIR_CHECK_MEMCAP(g_ippair_size))) {
             SCLogError(SC_ERR_IPPAIR_INIT, "preallocating ippairs failed: "
                     "max ippair memcap reached. Memcap %"PRIu64", "
                     "Memuse %"PRIu64".", ippair_config.memcap,
-                    ((uint64_t)SC_ATOMIC_GET(ippair_memuse) + (uint64_t)sizeof(IPPair)));
+                    ((uint64_t)SC_ATOMIC_GET(ippair_memuse) + g_ippair_size));
             exit(EXIT_FAILURE);
         }
 
@@ -226,8 +230,8 @@ void IPPairInitConfig(char quiet)
     }
 
     if (quiet == FALSE) {
-        SCLogInfo("preallocated %" PRIu32 " ippairs of size %" PRIuMAX "",
-                ippair_spare_q.len, (uintmax_t)sizeof(IPPair));
+        SCLogInfo("preallocated %" PRIu32 " ippairs of size %" PRIu16 "",
+                ippair_spare_q.len, g_ippair_size);
         SCLogInfo("ippair memory usage: %llu bytes, maximum: %"PRIu64,
                 SC_ATOMIC_GET(ippair_memuse), ippair_config.memcap);
     }
@@ -382,7 +386,7 @@ static IPPair *IPPairGetNew(Address *a, Address *b)
     h = IPPairDequeue(&ippair_spare_q);
     if (h == NULL) {
         /* If we reached the max memcap, we get a used ippair */
-        if (!(IPPAIR_CHECK_MEMCAP(sizeof(IPPair)))) {
+        if (!(IPPAIR_CHECK_MEMCAP(g_ippair_size))) {
             /* declare state of emergency */
             //if (!(SC_ATOMIC_GET(ippair_flags) & IPPAIR_EMERGENCY)) {
             //    SC_ATOMIC_OR(ippair_flags, IPPAIR_EMERGENCY);
diff --git a/framework/src/suricata/src/output-json-alert.c b/framework/src/suricata/src/output-json-alert.c
index 3c4219b4..2c0d0171 100644
--- a/framework/src/suricata/src/output-json-alert.c
+++ b/framework/src/suricata/src/output-json-alert.c
@@ -48,12 +48,15 @@
 #include "app-layer-htp-xff.h"
 #include "util-classification-config.h"
 #include "util-syslog.h"
+#include "util-logopenfile.h"
 
 #include "output.h"
 #include "output-json.h"
 #include "output-json-http.h"
 #include "output-json-tls.h"
 #include "output-json-ssh.h"
+#include "output-json-smtp.h"
+#include "output-json-email-common.h"
 
 #include "util-byte.h"
 #include "util-privs.h"
@@ -61,7 +64,6 @@
 #include "util-proto-name.h"
 #include "util-optimize.h"
 #include "util-buffer.h"
-#include "util-logopenfile.h"
 #include "util-crypt.h"
 
 #define MODULE_NAME "JsonAlertLog"
@@ -74,6 +76,7 @@
 #define LOG_JSON_HTTP 8
 #define LOG_JSON_TLS 16
 #define LOG_JSON_SSH 32
+#define LOG_JSON_SMTP 64
 
 #define JSON_STREAM_BUFFER_SIZE 4096
 
@@ -101,31 +104,6 @@ static int AlertJsonDumpStreamSegmentCallback(const Packet *p, void *data, uint8
     return 1;
 }
 
-/** Handle the case where no JSON support is compiled in.
- *
- */
-static void AlertJsonHttp(const Flow *f, json_t *js)
-{
-    HtpState *htp_state = (HtpState *)FlowGetAppState(f);
-    if (htp_state) {
-        uint64_t tx_id = AppLayerParserGetTransactionLogId(f->alparser);
-        htp_tx_t *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP, htp_state, tx_id);
-
-        if (tx) {
-            json_t *hjs = json_object();
-            if (unlikely(hjs == NULL))
-                return;
-
-            JsonHttpLogJSONBasic(hjs, tx);
-            JsonHttpLogJSONExtended(hjs, tx);
-
-            json_object_set_new(js, "http", hjs);
-        }
-    }
-
-    return;
-}
-
 static void AlertJsonTls(const Flow *f, json_t *js)
 {
     SSLState *ssl_state = (SSLState *)FlowGetAppState(f);
@@ -168,6 +146,11 @@ void AlertJsonHeader(const Packet *p, const PacketAlert *pa, json_t *js)
         action = "blocked";
     }
 
+    /* Add tx_id to root element for correlation with other events. */
+    json_object_del(js, "tx_id");
+    if (pa->flags & PACKET_ALERT_FLAG_TX)
+        json_object_set_new(js, "tx_id", json_integer(pa->tx_id));
+
     json_t *ajs = json_object();
     if (ajs == NULL) {
         json_decref(js);
@@ -184,9 +167,6 @@ void AlertJsonHeader(const Packet *p, const PacketAlert *pa, json_t *js)
             json_string((pa->s->class_msg) ? pa->s->class_msg : ""));
     json_object_set_new(ajs, "severity", json_integer(pa->s->prio));
 
-    if (pa->flags & PACKET_ALERT_FLAG_TX)
-        json_object_set_new(ajs, "tx_id", json_integer(pa->tx_id));
-
     if (p->tenant_id > 0)
         json_object_set_new(ajs, "tenant_id", json_integer(p->tenant_id));
 
@@ -198,6 +178,7 @@ static int AlertJson(ThreadVars *tv, JsonAlertLogThread *aft, const Packet *p)
 {
     MemBuffer *payload = aft->payload_buffer;
     AlertJsonOutputCtx *json_output_ctx = aft->json_output_ctx;
+    json_t *hjs = NULL;
 
     int i;
 
@@ -225,8 +206,11 @@ static int AlertJson(ThreadVars *tv, JsonAlertLogThread *aft, const Packet *p)
                 uint16_t proto = FlowGetAppProtocol(p->flow);
 
                 /* http alert */
-                if (proto == ALPROTO_HTTP)
-                    AlertJsonHttp(p->flow, js);
+                if (proto == ALPROTO_HTTP) {
+                    hjs = JsonHttpAddMetadata(p->flow, pa->tx_id);
+                    if (hjs)
+                        json_object_set_new(js, "http", hjs);
+                }
 
                 FLOWLOCK_UNLOCK(p->flow);
             }
@@ -258,6 +242,26 @@ static int AlertJson(ThreadVars *tv, JsonAlertLogThread *aft, const Packet *p)
             }
         }
 
+        if (json_output_ctx->flags & LOG_JSON_SMTP) {
+            if (p->flow != NULL) {
+                FLOWLOCK_RDLOCK(p->flow);
+                uint16_t proto = FlowGetAppProtocol(p->flow);
+
+                /* http alert */
+                if (proto == ALPROTO_SMTP) {
+                    hjs = JsonSMTPAddMetadata(p->flow, pa->tx_id);
+                    if (hjs)
+                        json_object_set_new(js, "smtp", hjs);
+
+                    hjs = JsonEmailAddMetadata(p->flow, pa->tx_id);
+                    if (hjs)
+                        json_object_set_new(js, "email", hjs);
+                }
+
+                FLOWLOCK_UNLOCK(p->flow);
+            }
+        }
+
         /* payload */
         if (json_output_ctx->flags & (LOG_JSON_PAYLOAD | LOG_JSON_PAYLOAD_BASE64)) {
             int stream = (p->proto == IPPROTO_TCP) ?
@@ -607,6 +611,7 @@ static OutputCtx *JsonAlertLogInitCtxSub(ConfNode *conf, OutputCtx *parent_ctx)
         const char *http = ConfNodeLookupChildValue(conf, "http");
         const char *tls = ConfNodeLookupChildValue(conf, "tls");
         const char *ssh = ConfNodeLookupChildValue(conf, "ssh");
+        const char *smtp = ConfNodeLookupChildValue(conf, "smtp");
 
         if (ssh != NULL) {
             if (ConfValIsTrue(ssh)) {
@@ -623,6 +628,11 @@ static OutputCtx *JsonAlertLogInitCtxSub(ConfNode *conf, OutputCtx *parent_ctx)
                 json_output_ctx->flags |= LOG_JSON_HTTP;
             }
         }
+        if (smtp != NULL) {
+            if (ConfValIsTrue(smtp)) {
+                json_output_ctx->flags |= LOG_JSON_SMTP;
+            }
+        }
         if (payload_printable != NULL) {
             if (ConfValIsTrue(payload_printable)) {
                 json_output_ctx->flags |= LOG_JSON_PAYLOAD;
diff --git a/framework/src/suricata/src/output-json-email-common.c b/framework/src/suricata/src/output-json-email-common.c
index 1efa9ce8..88cd3acf 100644
--- a/framework/src/suricata/src/output-json-email-common.c
+++ b/framework/src/suricata/src/output-json-email-common.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2014 Open Information Security Foundation
+/* Copyright (C) 2007-2015 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -19,6 +19,7 @@
  * \file
  *
  * \author Tom DeCanio <td@npulsetech.com>
+ * \author Eric Leblond <eric@regit.org>
  *
  * Implements json common email logging portion of the engine.
  */
@@ -55,137 +56,290 @@
 #ifdef HAVE_LIBJANSSON
 #include <jansson.h>
 
+#define LOG_EMAIL_DEFAULT       0
+#define LOG_EMAIL_EXTENDED      (1<<0)
+#define LOG_EMAIL_ARRAY         (1<<1) /* require array handling */
+#define LOG_EMAIL_COMMA         (1<<2) /* require array handling */
+#define LOG_EMAIL_BODY_MD5      (1<<3)
+#define LOG_EMAIL_SUBJECT_MD5   (1<<4)
+
+struct {
+    char *config_field;
+    char *email_field;
+    uint32_t flags;
+} email_fields[] =  {
+    { "reply_to", "reply-to", LOG_EMAIL_DEFAULT },
+    { "bcc", "bcc", LOG_EMAIL_COMMA|LOG_EMAIL_EXTENDED },
+    { "message_id", "message-id", LOG_EMAIL_EXTENDED },
+    { "subject", "subject", LOG_EMAIL_EXTENDED },
+    { "x_mailer", "x-mailer", LOG_EMAIL_EXTENDED },
+    { "user_agent", "user-agent", LOG_EMAIL_EXTENDED },
+    { "received", "received", LOG_EMAIL_ARRAY },
+    { "x_originating_ip", "x-originating-ip", LOG_EMAIL_DEFAULT },
+    { "in_reply_to",  "in-reply-to", LOG_EMAIL_DEFAULT },
+    { "references",  "references", LOG_EMAIL_DEFAULT },
+    { "importance",  "importance", LOG_EMAIL_DEFAULT },
+    { "priority",  "priority", LOG_EMAIL_DEFAULT },
+    { "sensitivity",  "sensitivity", LOG_EMAIL_DEFAULT },
+    { "organization",  "organization", LOG_EMAIL_DEFAULT },
+    { "content_md5",  "content-md5", LOG_EMAIL_DEFAULT },
+    { "date", "date", LOG_EMAIL_DEFAULT },
+    { NULL, NULL, LOG_EMAIL_DEFAULT},
+};
+
+static inline char *SkipWhiteSpaceTill(char *p, char *savep)
+{
+    char *sp = p;
+    if (unlikely(p == NULL)) {
+        return NULL;
+    }
+    while (((*sp == '\t') || (*sp == ' ')) && (sp < savep)) {
+        sp++;
+    }
+    return sp;
+}
+
+static json_t* JsonEmailJsonArrayFromCommaList(const uint8_t *val, size_t len)
+{
+    json_t *ajs = json_array();
+    if (likely(ajs != NULL)) {
+        char *savep = NULL;
+        char *p;
+        char *sp;
+        char *to_line = BytesToString((uint8_t *)val, len);
+        if (likely(to_line != NULL)) {
+            p = strtok_r(to_line, ",", &savep);
+            if (p == NULL) {
+                json_decref(ajs);
+                SCFree(to_line);
+                return NULL;
+            }
+            sp = SkipWhiteSpaceTill(p, savep);
+            json_array_append_new(ajs, json_string(sp));
+            while ((p = strtok_r(NULL, ",", &savep)) != NULL) {
+                sp = SkipWhiteSpaceTill(p, savep);
+                json_array_append_new(ajs, json_string(sp));
+            }
+        }
+        SCFree(to_line);
+    }
+
+    return ajs;
+}
+
+
+#ifdef HAVE_NSS
+static void JsonEmailLogJSONMd5(OutputJsonEmailCtx *email_ctx, json_t *js, SMTPTransaction *tx)
+{
+    if (email_ctx->flags & LOG_EMAIL_SUBJECT_MD5) {
+        MimeDecField *field;
+        MimeDecEntity *entity = tx->msg_tail;
+        if (entity == NULL) {
+            return;
+        }
+        field = MimeDecFindField(entity, "subject");
+        if (field != NULL) {
+            unsigned char md5[MD5_LENGTH];
+            char smd5[256];
+            char *value = BytesToString((uint8_t *)field->value , field->value_len);
+            if (value) {
+                size_t i,x;
+                HASH_HashBuf(HASH_AlgMD5, md5, (unsigned char *)value, strlen(value));
+                for (i = 0, x = 0; x < sizeof(md5); x++) {
+                    i += snprintf(smd5 + i, 255 - i, "%02x", md5[x]);
+                }
+                json_object_set_new(js, "subject_md5", json_string(smd5));
+                SCFree(value);
+            }
+        }
+    }
+
+    if (email_ctx->flags & LOG_EMAIL_BODY_MD5) {
+        MimeDecParseState *mime_state = tx->mime_state;
+        if (mime_state && mime_state->md5_ctx && (mime_state->state_flag == PARSE_DONE)) {
+            size_t x;
+            int i;
+            char s[256];
+            if (likely(s != NULL)) {
+                for (i = 0, x = 0; x < sizeof(mime_state->md5); x++) {
+                    i += snprintf(s + i, 255-i, "%02x", mime_state->md5[x]);
+                }
+                json_object_set_new(js, "body_md5", json_string(s));
+            }
+        }
+    }
+}
+#endif
+
+static int JsonEmailAddToJsonArray(const uint8_t *val, size_t len, void *data)
+{
+    json_t *ajs = data;
+
+    if (ajs == NULL)
+        return 0;
+    char *value = BytesToString((uint8_t *)val, len);
+    json_array_append_new(ajs, json_string(value));
+    SCFree(value);
+    return 1;
+}
+
+static void JsonEmailLogJSONCustom(OutputJsonEmailCtx *email_ctx, json_t *js, SMTPTransaction *tx)
+{
+    int f = 0;
+    MimeDecField *field;
+    MimeDecEntity *entity = tx->msg_tail;
+    if (entity == NULL) {
+        return;
+    }
+
+    while(email_fields[f].config_field) {
+        if (((email_ctx->fields & (1ULL<<f)) != 0)
+              ||
+              ((email_ctx->flags & LOG_EMAIL_EXTENDED) && (email_fields[f].flags & LOG_EMAIL_EXTENDED))
+           ) {
+            if (email_fields[f].flags & LOG_EMAIL_ARRAY) {
+                json_t *ajs = json_array();
+                if (ajs) {
+                    int found = MimeDecFindFieldsForEach(entity, email_fields[f].email_field, JsonEmailAddToJsonArray, ajs);
+                    if (found > 0) {
+                        json_object_set_new(js, email_fields[f].config_field, ajs);
+                    } else {
+                        json_decref(ajs);
+                    }
+                }
+            } else if (email_fields[f].flags & LOG_EMAIL_COMMA) {
+                field = MimeDecFindField(entity, email_fields[f].email_field);
+                if (field) {
+                    json_t *ajs = JsonEmailJsonArrayFromCommaList(field->value, field->value_len);
+                    if (ajs) {
+                        json_object_set_new(js, email_fields[f].config_field, ajs);
+                    }
+                }
+            } else {
+                field = MimeDecFindField(entity, email_fields[f].email_field);
+                if (field != NULL) {
+                    char *s = BytesToString((uint8_t *)field->value,
+                            (size_t)field->value_len);
+                    if (likely(s != NULL)) {
+                        json_object_set_new(js, email_fields[f].config_field, json_string(s));
+                        SCFree(s);
+                    }
+                }
+            }
+
+        }
+        f++;
+    }
+}
+
 /* JSON format logging */
-static TmEcode JsonEmailLogJson(JsonEmailLogThread *aft, json_t *js, const Packet *p, Flow *f, void *state, void *vtx, uint64_t tx_id)
+json_t *JsonEmailLogJsonData(const Flow *f, void *state, void *vtx, uint64_t tx_id)
 {
     SMTPState *smtp_state;
     MimeDecParseState *mime_state;
     MimeDecEntity *entity;
-    char *protos = NULL;
 
     json_t *sjs = json_object();
     if (sjs == NULL) {
-        SCReturnInt(TM_ECODE_FAILED);
+        SCReturnPtr(NULL, "json_t");
     }
 
     /* check if we have SMTP state or not */
-    AppProto proto = FlowGetAppProtocol(p->flow);
+    AppProto proto = FlowGetAppProtocol(f);
     switch (proto) {
         case ALPROTO_SMTP:
             smtp_state = (SMTPState *)state;
             if (smtp_state == NULL) {
                 SCLogDebug("no smtp state, so no request logging");
-                SCReturnInt(TM_ECODE_FAILED);
+                SCReturnPtr(NULL, "json_t");
             }
             SMTPTransaction *tx = vtx;
             mime_state = tx->mime_state;
             entity = tx->msg_tail;
-            protos = "smtp";
             SCLogDebug("lets go mime_state %p, entity %p, state_flag %u", mime_state, entity, mime_state ? mime_state->state_flag : 0);
             break;
         default:
             /* don't know how we got here */
-            SCReturnInt(TM_ECODE_FAILED);
+            SCReturnPtr(NULL, "json_t");
     }
     if ((mime_state != NULL)) {
         if (entity == NULL) {
-            SCReturnInt(TM_ECODE_FAILED);
+            SCReturnPtr(NULL, "json_t");
         }
 
-        if ((entity->header_flags & HDR_IS_LOGGED) == 0) {
-            MimeDecField *field;
-            //printf("email LOG\n");
-
-            /* From: */
-            field = MimeDecFindField(entity, "from");
-            if (field != NULL) {
-                char *s = BytesToString((uint8_t *)field->value,
-                                        (size_t)field->value_len);
-                if (likely(s != NULL)) {
-                    //printf("From: \"%s\"\n", s);
-                    json_object_set_new(sjs, "from", json_string(s));
-                    SCFree(s);
-                }
+        json_object_set_new(sjs, "status",
+                            json_string(MimeDecParseStateGetStatus(mime_state)));
+
+        MimeDecField *field;
+
+        /* From: */
+        field = MimeDecFindField(entity, "from");
+        if (field != NULL) {
+            char *s = BytesToString((uint8_t *)field->value,
+                                    (size_t)field->value_len);
+            if (likely(s != NULL)) {
+                //printf("From: \"%s\"\n", s);
+                char * sp = SkipWhiteSpaceTill(s, s + strlen(s));
+                json_object_set_new(sjs, "from", json_string(sp));
+                SCFree(s);
             }
+        }
 
-            /* To: */
-            char *to_line = NULL;
-            field = MimeDecFindField(entity, "to");
-            if (field != NULL) {
-                json_t *js_to = json_array();
-                if (likely(js_to != NULL)) {
-                    to_line = BytesToString((uint8_t *)field->value,
-                                            (size_t)field->value_len);
-                    if (likely(to_line != NULL)) {
-                        char *savep = NULL;
-                        char *p;
-                        //printf("to_line:: TO: \"%s\" (%d)\n", to_line, strlen(to_line));
-                        p = strtok_r(to_line, ",", &savep);
-                        //printf("got another addr: \"%s\"\n", p);
-                        json_array_append_new(js_to, json_string(p));
-                        while ((p = strtok_r(NULL, ",", &savep)) != NULL) {
-                            //printf("got another addr: \"%s\"\n", p);
-                            json_array_append_new(js_to, json_string(&p[strspn(p, " ")]));
-                        }
-                        SCFree(to_line);
-                    }
-                    json_object_set_new(sjs, "to", js_to);
-                }
+        /* To: */
+        field = MimeDecFindField(entity, "to");
+        if (field != NULL) {
+            json_t *ajs = JsonEmailJsonArrayFromCommaList(field->value, field->value_len);
+            if (ajs) {
+                json_object_set_new(sjs, "to", ajs);
             }
+        }
 
-            /* Cc: */
-            char *cc_line = NULL;
-            field = MimeDecFindField(entity, "cc");
-            if (field != NULL) {
-                json_t *js_cc = json_array();
-                if (likely(js_cc != NULL)) {
-                    cc_line = BytesToString((uint8_t *)field->value,
-                                            (size_t)field->value_len);
-                    if (likely(cc_line != NULL)) {
-                        char *savep = NULL;
-                        char *p;
-                        //printf("cc_line:: CC: \"%s\" (%d)\n", to_line, strlen(to_line));
-                        p = strtok_r(cc_line, ",", &savep);
-                        //printf("got another addr: \"%s\"\n", p);
-                        json_array_append_new(js_cc, json_string(p));
-                        while ((p = strtok_r(NULL, ",", &savep)) != NULL) {
-                            //printf("got another addr: \"%s\"\n", p);
-                            json_array_append_new(js_cc, json_string(&p[strspn(p, " ")]));
-                        }
-                        SCFree(cc_line);
-                    }
-                    json_object_set_new(sjs, "cc", js_cc);
-                }
+        /* Cc: */
+        field = MimeDecFindField(entity, "cc");
+        if (field != NULL) {
+            json_t *ajs = JsonEmailJsonArrayFromCommaList(field->value, field->value_len);
+            if (ajs) {
+                json_object_set_new(sjs, "cc", ajs);
             }
+        }
 
-            /* Subject: */
-            field = MimeDecFindField(entity, "subject");
-            if (field != NULL) {
-                char *s = BytesToString((uint8_t *)field->value, (size_t) field->value_len);
-                if (likely(s != NULL)) {
-                    //printf("Subject: \"%s\"\n", s);
-                    json_object_set_new(sjs, "subject", json_string(s));
+        if (mime_state->stack == NULL || mime_state->stack->top == NULL || mime_state->stack->top->data == NULL)
+            SCReturnPtr(NULL, "json_t");
+
+        entity = (MimeDecEntity *)mime_state->stack->top->data;
+        int attch_cnt = 0;
+        int url_cnt = 0;
+        json_t *js_attch = json_array();
+        json_t *js_url = json_array();
+        if (entity->url_list != NULL) {
+            MimeDecUrl *url;
+            for (url = entity->url_list; url != NULL; url = url->next) {
+                char *s = BytesToString((uint8_t *)url->url,
+                                        (size_t)url->url_len);
+                if (s != NULL) {
+                    json_array_append_new(js_url,
+                                      json_string(s));
                     SCFree(s);
+                    url_cnt += 1;
                 }
             }
+        }
+        for (entity = entity->child; entity != NULL; entity = entity->next) {
+            if (entity->ctnt_flags & CTNT_IS_ATTACHMENT) {
 
-            entity->header_flags |= HDR_IS_LOGGED;
-
-            if (mime_state->stack == NULL || mime_state->stack->top == NULL || mime_state->stack->top->data == NULL)
-                SCReturnInt(TM_ECODE_OK);
-
-            entity = (MimeDecEntity *)mime_state->stack->top->data;
-            int attch_cnt = 0;
-            int url_cnt = 0;
-            json_t *js_attch = json_array();
-            json_t *js_url = json_array();
+                char *s = BytesToString((uint8_t *)entity->filename,
+                                        (size_t)entity->filename_len);
+                json_array_append_new(js_attch,
+                                      json_string(s));
+                SCFree(s);
+                attch_cnt += 1;
+            }
             if (entity->url_list != NULL) {
                 MimeDecUrl *url;
                 for (url = entity->url_list; url != NULL; url = url->next) {
                     char *s = BytesToString((uint8_t *)url->url,
                                             (size_t)url->url_len);
                     if (s != NULL) {
-                        //printf("URL: \"%s\"\n", s);
                         json_array_append_new(js_url,
                                           json_string(s));
                         SCFree(s);
@@ -193,74 +347,113 @@ static TmEcode JsonEmailLogJson(JsonEmailLogThread *aft, json_t *js, const Packe
                     }
                 }
             }
-            for (entity = entity->child; entity != NULL; entity = entity->next) {
-                if (entity->ctnt_flags & CTNT_IS_ATTACHMENT) {
-
-                    char *s = BytesToString((uint8_t *)entity->filename,
-                                            (size_t)entity->filename_len);
-                    //printf("found attachment \"%s\"\n", s);
-                    json_array_append_new(js_attch,
-                                          json_string(s));
-                    SCFree(s);
-                    attch_cnt += 1;
-                }
-                if (entity->url_list != NULL) {
-                    MimeDecUrl *url;
-                    for (url = entity->url_list; url != NULL; url = url->next) {
-                        char *s = BytesToString((uint8_t *)url->url,
-                                                (size_t)url->url_len);
-                        if (s != NULL) {
-                            //printf("URL: \"%s\"\n", s);
-                            json_array_append_new(js_url,
-                                              json_string(s));
-                            SCFree(s);
-                            url_cnt += 1;
-                        }
-                    }
-                }
-            }
-            if (attch_cnt > 0) {
-                json_object_set_new(sjs, "attachment", js_attch);
-            } else {
-                json_decref(js_attch);
-            }
-            if (url_cnt > 0) {
-                json_object_set_new(sjs, "url", js_url);
-            } else {
-                json_decref(js_url);
-            }
-            json_object_set_new(js, protos, sjs);
-
-//            FLOWLOCK_UNLOCK(p->flow);
-            SCReturnInt(TM_ECODE_OK);
         }
+        if (attch_cnt > 0) {
+            json_object_set_new(sjs, "attachment", js_attch);
+        } else {
+            json_decref(js_attch);
+        }
+        if (url_cnt > 0) {
+            json_object_set_new(sjs, "url", js_url);
+        } else {
+            json_decref(js_url);
+        }
+        SCReturnPtr(sjs, "json_t");
     }
 
-//    FLOWLOCK_UNLOCK(p->flow);
-    SCReturnInt(TM_ECODE_DONE);
+    json_decref(sjs);
+    SCReturnPtr(NULL, "json_t");
 }
 
-int JsonEmailLogger(ThreadVars *tv, void *thread_data, const Packet *p, Flow *f, void *state, void *tx, uint64_t tx_id) {
-    SCEnter();
-    JsonEmailLogThread *jhl = (JsonEmailLogThread *)thread_data;
-    MemBuffer *buffer = (MemBuffer *)jhl->buffer;
+/* JSON format logging */
+TmEcode JsonEmailLogJson(JsonEmailLogThread *aft, json_t *js, const Packet *p, Flow *f, void *state, void *vtx, uint64_t tx_id)
+{
+    json_t *sjs = JsonEmailLogJsonData(f, state, vtx, tx_id);
+    OutputJsonEmailCtx *email_ctx = aft->emaillog_ctx;
+    SMTPTransaction *tx = (SMTPTransaction *) vtx;
+
+    if ((email_ctx->flags & LOG_EMAIL_EXTENDED) || (email_ctx->fields != 0))
+        JsonEmailLogJSONCustom(email_ctx, sjs, tx);
 
-    json_t *js = CreateJSONHeader((Packet *)p, 1, "smtp");
-    if (unlikely(js == NULL))
-        return TM_ECODE_OK;
+#ifdef HAVE_NSS
+    JsonEmailLogJSONMd5(email_ctx, sjs, tx);
+#endif
 
-    /* reset */
-    MemBufferReset(buffer);
+    if (sjs) {
+        json_object_set_new(js, "email", sjs);
+        SCReturnInt(TM_ECODE_OK);
+    } else
+        SCReturnInt(TM_ECODE_FAILED);
+}
+
+json_t *JsonEmailAddMetadata(const Flow *f, uint32_t tx_id)
+{
+    SMTPState *smtp_state = (SMTPState *)FlowGetAppState(f);
+    if (smtp_state) {
+        SMTPTransaction *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_SMTP, smtp_state, tx_id);
 
-    if (JsonEmailLogJson(jhl, js, p, f, state, tx, tx_id) == TM_ECODE_OK) {
-        OutputJSONBuffer(js, jhl->emaillog_ctx->file_ctx, buffer);
+        if (tx) {
+            return JsonEmailLogJsonData(f, smtp_state, tx, tx_id);
+        }
     }
-    json_object_del(js, "smtp");
 
-    json_object_clear(js);
-    json_decref(js);
+    return NULL;
+}
+
+
+void OutputEmailInitConf(ConfNode *conf, OutputJsonEmailCtx *email_ctx)
+{
+    if (conf) {
+        const char *extended = ConfNodeLookupChildValue(conf, "extended");
+
+        if (extended != NULL) {
+            if (ConfValIsTrue(extended)) {
+                email_ctx->flags = LOG_EMAIL_EXTENDED;
+            }
+        }
+
+        email_ctx->fields  = 0;
+        ConfNode *custom;
+        if ((custom = ConfNodeLookupChild(conf, "custom")) != NULL) {
+            ConfNode *field;
+            TAILQ_FOREACH(field, &custom->head, next) {
+                if (field != NULL) {
+                    int f = 0;
+                    while(email_fields[f].config_field) {
+                        if ((strcmp(email_fields[f].config_field,
+                                   field->val) == 0) ||
+                            (strcasecmp(email_fields[f].email_field,
+                                        field->val) == 0))
+                        {
+                            email_ctx->fields |= (1ULL<<f);
+                            break;
+                        }
+                        f++;
+                    }
+                }
+            }
+        }
 
-    SCReturnInt(TM_ECODE_OK);
+        email_ctx->flags  = 0;
+        ConfNode *md5_conf;
+        if ((md5_conf = ConfNodeLookupChild(conf, "md5")) != NULL) {
+            ConfNode *field;
+            TAILQ_FOREACH(field, &md5_conf->head, next) {
+                if (field != NULL) {
+                    if (strcmp("body", field->val) == 0) {
+                        SCLogInfo("Going to log the md5 sum of email body");
+                        email_ctx->flags |= LOG_EMAIL_BODY_MD5;
+                    }
+                    if (strcmp("subject", field->val) == 0) {
+                        SCLogInfo("Going to log the md5 sum of email subject");
+                        email_ctx->flags |= LOG_EMAIL_SUBJECT_MD5;
+                    }
+                }
+            }
+        }
+    }
+    return;
 }
 
+
 #endif
diff --git a/framework/src/suricata/src/output-json-email-common.h b/framework/src/suricata/src/output-json-email-common.h
index 7a95954c..88cfa557 100644
--- a/framework/src/suricata/src/output-json-email-common.h
+++ b/framework/src/suricata/src/output-json-email-common.h
@@ -27,14 +27,20 @@
 typedef struct OutputJsonEmailCtx_ {
     LogFileCtx *file_ctx;
     uint32_t flags; /** Store mode */
+    uint64_t fields;/** Store fields */
 } OutputJsonEmailCtx;
 
 
+#ifdef HAVE_LIBJANSSON
 typedef struct JsonEmailLogThread_ {
     OutputJsonEmailCtx *emaillog_ctx;
     MemBuffer *buffer;
 } JsonEmailLogThread;
 
-int JsonEmailLogger(ThreadVars *tv, void *thread_data, const Packet *p, Flow *f, void *state, void *tx, uint64_t tx_id);
+TmEcode JsonEmailLogJson(JsonEmailLogThread *aft, json_t *js, const Packet *p, Flow *f, void *state, void *vtx, uint64_t tx_id);
+json_t *JsonEmailAddMetadata(const Flow *f, uint32_t tx_id);
+#endif
+
+void OutputEmailInitConf(ConfNode *conf, OutputJsonEmailCtx *email_ctx);
 
 #endif /* __OUTPUT_JSON_EMAIL_COMMON_H__ */
diff --git a/framework/src/suricata/src/output-json-file.c b/framework/src/suricata/src/output-json-file.c
index cbfa0c4d..9506a746 100644
--- a/framework/src/suricata/src/output-json-file.c
+++ b/framework/src/suricata/src/output-json-file.c
@@ -51,12 +51,15 @@
 #include "util-buffer.h"
 #include "util-byte.h"
 
-#include "output.h"
-#include "output-json.h"
-
 #include "log-file.h"
 #include "util-logopenfile.h"
 
+#include "output.h"
+#include "output-json.h"
+#include "output-json-http.h"
+#include "output-json-smtp.h"
+#include "output-json-email-common.h"
+
 #include "app-layer-htp.h"
 #include "util-memcmp.h"
 #include "stream-tcp-reassemble.h"
@@ -74,99 +77,6 @@ typedef struct JsonFileLogThread_ {
     MemBuffer *buffer;
 } JsonFileLogThread;
 
-static json_t *LogFileMetaGetUri(const Packet *p, const File *ff)
-{
-    HtpState *htp_state = (HtpState *)p->flow->alstate;
-    json_t *js = NULL;
-    if (htp_state != NULL) {
-        htp_tx_t *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP, htp_state, ff->txid);
-        if (tx != NULL) {
-            HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
-            if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
-                char *s = bstr_util_strdup_to_c(tx_ud->request_uri_normalized);
-                if (s != NULL) {
-                    js = json_string(s);
-                    SCFree(s);
-                    if (js != NULL)
-                        return js;
-                }
-            }
-        }
-    }
-
-    return NULL;
-}
-
-static json_t *LogFileMetaGetHost(const Packet *p, const File *ff)
-{
-    HtpState *htp_state = (HtpState *)p->flow->alstate;
-    json_t *js = NULL;
-    if (htp_state != NULL) {
-        htp_tx_t *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP, htp_state, ff->txid);
-        if (tx != NULL && tx->request_hostname != NULL) {
-            char *s = bstr_util_strdup_to_c(tx->request_hostname);
-            if (s != NULL) {
-                js = json_string(s);
-                SCFree(s);
-                if (js != NULL)
-                    return js;
-            }
-        }
-    }
-
-    return NULL;
-}
-
-static json_t *LogFileMetaGetReferer(const Packet *p, const File *ff)
-{
-    HtpState *htp_state = (HtpState *)p->flow->alstate;
-    json_t *js = NULL;
-    if (htp_state != NULL) {
-        htp_tx_t *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP, htp_state, ff->txid);
-        if (tx != NULL) {
-            htp_header_t *h = NULL;
-            h = (htp_header_t *)htp_table_get_c(tx->request_headers,
-                                                "Referer");
-            if (h != NULL) {
-                char *s = bstr_util_strdup_to_c(h->value);
-                if (s != NULL) {
-                    js = json_string(s);
-                    SCFree(s);
-                    if (js != NULL)
-                        return js;
-                }
-            }
-        }
-    }
-
-    return NULL;
-}
-
-static json_t *LogFileMetaGetUserAgent(const Packet *p, const File *ff)
-{
-    HtpState *htp_state = (HtpState *)p->flow->alstate;
-    json_t *js = NULL;
-    if (htp_state != NULL) {
-        htp_tx_t *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP, htp_state, ff->txid);
-        if (tx != NULL) {
-            htp_header_t *h = NULL;
-            h = (htp_header_t *)htp_table_get_c(tx->request_headers,
-                                                "User-Agent");
-            if (h != NULL) {
-                char *s = bstr_util_strdup_to_c(h->value);
-                if (s != NULL) {
-                    js = json_string(s);
-                    SCFree(s);
-                    if (js != NULL)
-                        return js;
-                }
-            }
-        }
-    }
-
-    return NULL;
-}
-
 /**
  *  \internal
  *  \brief Write meta data on a single line json record
@@ -175,33 +85,32 @@ static void FileWriteJsonRecord(JsonFileLogThread *aft, const Packet *p, const F
 {
     MemBuffer *buffer = (MemBuffer *)aft->buffer;
     json_t *js = CreateJSONHeader((Packet *)p, 0, "fileinfo"); //TODO const
+    json_t *hjs = NULL;
     if (unlikely(js == NULL))
         return;
 
     /* reset */
     MemBufferReset(buffer);
 
-    json_t *hjs = json_object();
-    if (unlikely(hjs == NULL)) {
-        json_decref(js);
-        return;
-    }
-
-    json_object_set_new(hjs, "app_proto", json_string(AppProtoToString(p->flow->alproto)));
     switch (p->flow->alproto) {
         case ALPROTO_HTTP:
-            json_object_set_new(hjs, "url", LogFileMetaGetUri(p, ff));
-            json_object_set_new(hjs, "hostname", LogFileMetaGetHost(p, ff));
-            json_object_set_new(hjs, "http_refer", LogFileMetaGetReferer(p, ff));
-            json_object_set_new(hjs, "http_user_agent", LogFileMetaGetUserAgent(p, ff));
-            json_object_set_new(js, "http", hjs);
+            hjs = JsonHttpAddMetadata(p->flow, ff->txid);
+            if (hjs)
+                json_object_set_new(js, "http", hjs);
+            break;
+        case ALPROTO_SMTP:
+            hjs = JsonSMTPAddMetadata(p->flow, ff->txid);
+            if (hjs)
+                json_object_set_new(js, "smtp", hjs);
+            hjs = JsonEmailAddMetadata(p->flow, ff->txid);
+            if (hjs)
+                json_object_set_new(js, "email", hjs);
             break;
     }
 
 
     json_t *fjs = json_object();
     if (unlikely(fjs == NULL)) {
-        json_decref(hjs);
         json_decref(js);
         return;
     }
@@ -219,14 +128,11 @@ static void FileWriteJsonRecord(JsonFileLogThread *aft, const Packet *p, const F
             if (ff->flags & FILE_MD5) {
                 size_t x;
                 int i;
-                char *s = SCMalloc(256);
-                if (likely(s != NULL)) {
-                    for (i = 0, x = 0; x < sizeof(ff->md5); x++) {
-                        i += snprintf(&s[i], 255-i, "%02x", ff->md5[x]);
-                    }
-                    json_object_set_new(fjs, "md5", json_string(s));
-                    SCFree(s);
+                char s[256];
+                for (i = 0, x = 0; x < sizeof(ff->md5); x++) {
+                    i += snprintf(&s[i], 255-i, "%02x", ff->md5[x]);
                 }
+                json_object_set_new(fjs, "md5", json_string(s));
             }
 #endif
             break;
@@ -252,7 +158,16 @@ static void FileWriteJsonRecord(JsonFileLogThread *aft, const Packet *p, const F
     json_object_set_new(js, "fileinfo", fjs);
     OutputJSONBuffer(js, aft->filelog_ctx->file_ctx, buffer);
     json_object_del(js, "fileinfo");
-    json_object_del(js, "http");
+
+    switch (p->flow->alproto) {
+        case ALPROTO_HTTP:
+            json_object_del(js, "http");
+            break;
+        case ALPROTO_SMTP:
+            json_object_del(js, "smtp");
+            json_object_del(js, "email");
+            break;
+    }
 
     json_object_clear(js);
     json_decref(js);
diff --git a/framework/src/suricata/src/output-json-http.c b/framework/src/suricata/src/output-json-http.c
index 31641985..68739873 100644
--- a/framework/src/suricata/src/output-json-http.c
+++ b/framework/src/suricata/src/output-json-http.c
@@ -362,9 +362,6 @@ static void JsonHttpLogJSON(JsonHttpLogThread *aft, json_t *js, htp_tx_t *tx, ui
     if (http_ctx->flags & LOG_HTTP_EXTENDED)
         JsonHttpLogJSONExtended(hjs, tx);
 
-    /* tx id for correlation with alerts */
-    json_object_set_new(hjs, "tx_id", json_integer(tx_id));
-
     json_object_set_new(js, "http", hjs);
 }
 
@@ -376,7 +373,7 @@ static int JsonHttpLogger(ThreadVars *tv, void *thread_data, const Packet *p, Fl
     JsonHttpLogThread *jhl = (JsonHttpLogThread *)thread_data;
     MemBuffer *buffer = (MemBuffer *)jhl->buffer;
 
-    json_t *js = CreateJSONHeader((Packet *)p, 1, "http"); //TODO const
+    json_t *js = CreateJSONHeaderWithTxId((Packet *)p, 1, "http", tx_id); //TODO const
     if (unlikely(js == NULL))
         return TM_ECODE_OK;
 
@@ -396,6 +393,27 @@ static int JsonHttpLogger(ThreadVars *tv, void *thread_data, const Packet *p, Fl
     SCReturnInt(TM_ECODE_OK);
 }
 
+json_t *JsonHttpAddMetadata(const Flow *f, uint64_t tx_id)
+{
+    HtpState *htp_state = (HtpState *)FlowGetAppState(f);
+    if (htp_state) {
+        htp_tx_t *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP, htp_state, tx_id);
+
+        if (tx) {
+            json_t *hjs = json_object();
+            if (unlikely(hjs == NULL))
+                return NULL;
+
+            JsonHttpLogJSONBasic(hjs, tx);
+            JsonHttpLogJSONExtended(hjs, tx);
+
+            return hjs;
+        }
+    }
+
+    return NULL;
+}
+
 static void OutputHttpLogDeinit(OutputCtx *output_ctx)
 {
     LogHttpFileCtx *http_ctx = output_ctx->data;
diff --git a/framework/src/suricata/src/output-json-http.h b/framework/src/suricata/src/output-json-http.h
index ab412d22..0c886f3d 100644
--- a/framework/src/suricata/src/output-json-http.h
+++ b/framework/src/suricata/src/output-json-http.h
@@ -29,6 +29,7 @@ void TmModuleJsonHttpLogRegister (void);
 #ifdef HAVE_LIBJANSSON
 void JsonHttpLogJSONBasic(json_t *js, htp_tx_t *tx);
 void JsonHttpLogJSONExtended(json_t *js, htp_tx_t *tx);
+json_t *JsonHttpAddMetadata(const Flow *f, uint64_t tx_id);
 #endif /* HAVE_LIBJANSSON */
 
 #endif /* __OUTPUT_JSON_HTTP_H__ */
diff --git a/framework/src/suricata/src/output-json-smtp.c b/framework/src/suricata/src/output-json-smtp.c
index f722c383..617b7247 100644
--- a/framework/src/suricata/src/output-json-smtp.c
+++ b/framework/src/suricata/src/output-json-smtp.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2013 Open Information Security Foundation
+/* Copyright (C) 2007-2015 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -54,11 +54,81 @@
 #ifdef HAVE_LIBJANSSON
 #include <jansson.h>
 
+static json_t *JsonSmtpDataLogger(const Flow *f, void *state, void *vtx, uint64_t tx_id)
+{
+    json_t *sjs = json_object();
+    SMTPTransaction *tx = vtx;
+    SMTPString *rcptto_str;
+    if (sjs == NULL) {
+        return NULL;
+    }
+    if (((SMTPState *)state)->helo) {
+        json_object_set_new(sjs, "helo",
+                            json_string((const char *)((SMTPState *)state)->helo));
+    }
+    if (tx->mail_from) {
+        json_object_set_new(sjs, "mail_from",
+                            json_string((const char *)tx->mail_from));
+    }
+    if (!TAILQ_EMPTY(&tx->rcpt_to_list)) {
+        json_t *js_rcptto = json_array();
+        if (likely(js_rcptto != NULL)) {
+            TAILQ_FOREACH(rcptto_str, &tx->rcpt_to_list, next) {
+                json_array_append_new(js_rcptto, json_string((char *)rcptto_str->str));
+            }
+            json_object_set_new(sjs, "rcpt_to", js_rcptto);
+        }
+    }
+
+    return sjs;
+}
+
 static int JsonSmtpLogger(ThreadVars *tv, void *thread_data, const Packet *p, Flow *f, void *state, void *tx, uint64_t tx_id)
 {
     SCEnter();
-    int r = JsonEmailLogger(tv, thread_data, p, f, state, tx, tx_id);
-    SCReturnInt(r);
+    JsonEmailLogThread *jhl = (JsonEmailLogThread *)thread_data;
+    MemBuffer *buffer = (MemBuffer *)jhl->buffer;
+
+    json_t *sjs;
+    json_t *js = CreateJSONHeaderWithTxId((Packet *)p, 1, "smtp", tx_id);
+    if (unlikely(js == NULL))
+        return TM_ECODE_OK;
+
+    /* reset */
+    MemBufferReset(buffer);
+
+    sjs = JsonSmtpDataLogger(f, state, tx, tx_id);
+    if (sjs) {
+        json_object_set_new(js, "smtp", sjs);
+    }
+
+    if (JsonEmailLogJson(jhl, js, p, f, state, tx, tx_id) == TM_ECODE_OK) {
+        OutputJSONBuffer(js, jhl->emaillog_ctx->file_ctx, buffer);
+    }
+    json_object_del(js, "email");
+    if (sjs) {
+        json_object_del(js, "smtp");
+    }
+
+    json_object_clear(js);
+    json_decref(js);
+
+    SCReturnInt(TM_ECODE_OK);
+
+}
+
+json_t *JsonSMTPAddMetadata(const Flow *f, uint64_t tx_id)
+{
+    SMTPState *smtp_state = (SMTPState *)FlowGetAppState(f);
+    if (smtp_state) {
+        SMTPTransaction *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_SMTP, smtp_state, tx_id);
+
+        if (tx) {
+            return JsonSmtpDataLogger(f, smtp_state, tx, tx_id);
+        }
+    }
+
+    return NULL;
 }
 
 static void OutputSmtpLogDeInitCtx(OutputCtx *output_ctx)
@@ -135,6 +205,8 @@ static OutputCtx *OutputSmtpLogInitSub(ConfNode *conf, OutputCtx *parent_ctx)
 
     email_ctx->file_ctx = ojc->file_ctx;
 
+    OutputEmailInitConf(conf, email_ctx);
+
     output_ctx->data = email_ctx;
     output_ctx->DeInit = OutputSmtpLogDeInitCtxSub;
 
@@ -154,7 +226,7 @@ static TmEcode JsonSmtpLogThreadInit(ThreadVars *t, void *initdata, void **data)
 
     if(initdata == NULL)
     {
-        SCLogDebug("Error getting context for HTTPLog.  \"initdata\" argument NULL");
+        SCLogDebug("Error getting context for SMTPLog.  \"initdata\" argument NULL");
         SCFree(aft);
         return TM_ECODE_FAILED;
     }
diff --git a/framework/src/suricata/src/output-json-smtp.h b/framework/src/suricata/src/output-json-smtp.h
index d38187c7..2f79d992 100644
--- a/framework/src/suricata/src/output-json-smtp.h
+++ b/framework/src/suricata/src/output-json-smtp.h
@@ -25,5 +25,8 @@
 #define __OUTPUT_JSON_SMTP_H__
 
 void TmModuleJsonSmtpLogRegister (void);
+#ifdef HAVE_LIBJANSSON
+json_t *JsonSMTPAddMetadata(const Flow *f, uint64_t tx_id);
+#endif
 
 #endif /* __OUTPUT_JSON_SMTP_H__ */
diff --git a/framework/src/suricata/src/output-json-template.c b/framework/src/suricata/src/output-json-template.c
index ca4a9378..d360e674 100644
--- a/framework/src/suricata/src/output-json-template.c
+++ b/framework/src/suricata/src/output-json-template.c
@@ -180,6 +180,10 @@ static TmEcode JsonTemplateLogThreadDeinit(ThreadVars *t, void *data)
 
 void TmModuleJsonTemplateLogRegister(void)
 {
+    if (ConfGetNode("app-layer.protocols.template") == NULL) {
+        return;
+    }
+
     tmm_modules[TMM_JSONTEMPLATELOG].name = "JsonTemplateLog";
     tmm_modules[TMM_JSONTEMPLATELOG].ThreadInit = JsonTemplateLogThreadInit;
     tmm_modules[TMM_JSONTEMPLATELOG].ThreadDeinit = JsonTemplateLogThreadDeinit;
diff --git a/framework/src/suricata/src/output-json.c b/framework/src/suricata/src/output-json.c
index 74289f1b..9cc9bd94 100644
--- a/framework/src/suricata/src/output-json.c
+++ b/framework/src/suricata/src/output-json.c
@@ -119,10 +119,6 @@ void OutputJsonRegisterTests (void)
 
 #define OUTPUT_BUFFER_SIZE 65535
 
-#ifndef OS_WIN32
-static int alert_syslog_level = DEFAULT_ALERT_SYSLOG_LEVEL;
-#endif /* OS_WIN32 */
-
 TmEcode OutputJson (ThreadVars *, Packet *, void *, PacketQueue *, PacketQueue *);
 TmEcode OutputJsonThreadInit(ThreadVars *, void *, void **);
 TmEcode OutputJsonThreadDeinit(ThreadVars *, void *);
@@ -327,48 +323,51 @@ json_t *CreateJSONHeader(Packet *p, int direction_sensitive, char *event_type)
     return js;
 }
 
+json_t *CreateJSONHeaderWithTxId(Packet *p, int direction_sensitive, char *event_type, uint32_t tx_id)
+{
+    json_t *js = CreateJSONHeader(p, direction_sensitive, event_type);
+    if (unlikely(js == NULL))
+        return NULL;
+
+    /* tx id for correlation with other events */
+    json_object_set_new(js, "tx_id", json_integer(tx_id));
+
+    return js;
+}
+
+static int MemBufferCallback(const char *str, size_t size, void *data)
+{
+    MemBuffer *memb = data;
+#if 0 // can't expand, need a MemBuffer **
+    /* since we can have many threads, the buffer might not be big enough.
+     *              * Expand if necessary. */
+    if (MEMBUFFER_OFFSET(memb) + size > MEMBUFFER_SIZE(memb)) {
+        MemBufferExpand(&memb, OUTPUT_BUFFER_SIZE);
+    }
+#endif
+    MemBufferWriteString(memb, "%s", str);
+    return 0;
+}
+
 int OutputJSONBuffer(json_t *js, LogFileCtx *file_ctx, MemBuffer *buffer)
 {
-    char *js_s = json_dumps(js,
-                            JSON_PRESERVE_ORDER|JSON_COMPACT|JSON_ENSURE_ASCII|
+    if (file_ctx->sensor_name) {
+        json_object_set_new(js, "host",
+                            json_string(file_ctx->sensor_name));
+    }
+
+    int r = json_dump_callback(js, MemBufferCallback, buffer,
+            JSON_PRESERVE_ORDER|JSON_COMPACT|JSON_ENSURE_ASCII|
 #ifdef JSON_ESCAPE_SLASH
                             JSON_ESCAPE_SLASH
 #else
                             0
 #endif
                             );
-    if (unlikely(js_s == NULL))
+    if (r != 0)
         return TM_ECODE_OK;
 
-    SCMutexLock(&file_ctx->fp_mutex);
-    if (file_ctx->type == LOGFILE_TYPE_SYSLOG)
-    {
-        if (file_ctx->prefix != NULL)
-        {
-            syslog(alert_syslog_level, "%s%s", file_ctx->prefix, js_s);
-        }
-        else
-        {
-            syslog(alert_syslog_level, "%s", js_s);
-        }
-    }
-    else if (file_ctx->type == LOGFILE_TYPE_FILE ||
-               file_ctx->type == LOGFILE_TYPE_UNIX_DGRAM ||
-               file_ctx->type == LOGFILE_TYPE_UNIX_STREAM)
-    {
-        if (file_ctx->prefix != NULL)
-        {
-            MemBufferWriteString(buffer, "%s%s\n", file_ctx->prefix, js_s);
-        }
-        else
-        {
-            MemBufferWriteString(buffer, "%s\n", js_s);
-        }
-        file_ctx->Write((const char *)MEMBUFFER_BUFFER(buffer),
-            MEMBUFFER_OFFSET(buffer), file_ctx);
-    }
-    SCMutexUnlock(&file_ctx->fp_mutex);
-    free(js_s);
+    LogFileWrite(file_ctx, buffer);
     return 0;
 }
 
@@ -425,6 +424,9 @@ void OutputJsonExitPrintStats(ThreadVars *tv, void *data)
 OutputCtx *OutputJsonInitCtx(ConfNode *conf)
 {
     OutputJsonCtx *json_ctx = SCCalloc(1, sizeof(OutputJsonCtx));;
+
+    const char *sensor_name = ConfNodeLookupChildValue(conf, "sensor-name");
+
     if (unlikely(json_ctx == NULL)) {
         SCLogDebug("AlertJsonInitCtx: Could not create new LogFileCtx");
         return NULL;
@@ -437,6 +439,17 @@ OutputCtx *OutputJsonInitCtx(ConfNode *conf)
         return NULL;
     }
 
+    if (sensor_name) {
+        json_ctx->file_ctx->sensor_name = SCStrdup(sensor_name);
+        if (json_ctx->file_ctx->sensor_name  == NULL) {
+            LogFileFreeCtx(json_ctx->file_ctx);
+            SCFree(json_ctx);
+            return NULL;
+        }
+    } else {
+        json_ctx->file_ctx->sensor_name = NULL;
+    }
+
     OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
     if (unlikely(output_ctx == NULL)) {
         LogFileFreeCtx(json_ctx->file_ctx);
@@ -465,6 +478,14 @@ OutputCtx *OutputJsonInitCtx(ConfNode *conf)
                 json_ctx->json_out = LOGFILE_TYPE_UNIX_DGRAM;
             } else if (strcmp(output_s, "unix_stream") == 0) {
                 json_ctx->json_out = LOGFILE_TYPE_UNIX_STREAM;
+            } else if (strcmp(output_s, "redis") == 0) {
+#ifdef HAVE_LIBHIREDIS
+                json_ctx->json_out = LOGFILE_TYPE_REDIS;
+#else
+                SCLogError(SC_ERR_INVALID_ARGUMENT,
+                           "redis JSON output option is not compiled");
+                exit(EXIT_FAILURE);
+#endif
             } else {
                 SCLogError(SC_ERR_INVALID_ARGUMENT,
                            "Invalid JSON output option: %s", output_s);
@@ -526,7 +547,7 @@ OutputCtx *OutputJsonInitCtx(ConfNode *conf)
             if (level_s != NULL) {
                 int level = SCMapEnumNameToValue(level_s, SCSyslogGetLogLevelMap());
                 if (level != -1) {
-                    alert_syslog_level = level;
+                    json_ctx->file_ctx->syslog_setup.alert_syslog_level = level;
                 }
             }
 
@@ -537,6 +558,29 @@ OutputCtx *OutputJsonInitCtx(ConfNode *conf)
             openlog(ident, LOG_PID|LOG_NDELAY, facility);
 
         }
+#ifdef HAVE_LIBHIREDIS
+        else if (json_ctx->json_out == LOGFILE_TYPE_REDIS) {
+            ConfNode *redis_node = ConfNodeLookupChild(conf, "redis");
+            if (!json_ctx->file_ctx->sensor_name) {
+                char hostname[1024];
+                gethostname(hostname, 1023);
+                json_ctx->file_ctx->sensor_name = SCStrdup(hostname);
+            }
+            if (json_ctx->file_ctx->sensor_name  == NULL) {
+                LogFileFreeCtx(json_ctx->file_ctx);
+                SCFree(json_ctx);
+                SCFree(output_ctx);
+                return NULL;
+            }
+
+            if (SCConfLogOpenRedis(redis_node, json_ctx->file_ctx) < 0) {
+                LogFileFreeCtx(json_ctx->file_ctx);
+                SCFree(json_ctx);
+                SCFree(output_ctx);
+                return NULL;
+            }
+        }
+#endif
 
         const char *sensor_id_s = ConfNodeLookupChildValue(conf, "sensor-id");
         if (sensor_id_s != NULL) {
diff --git a/framework/src/suricata/src/output-json.h b/framework/src/suricata/src/output-json.h
index 1acde3e6..89e11d86 100644
--- a/framework/src/suricata/src/output-json.h
+++ b/framework/src/suricata/src/output-json.h
@@ -35,11 +35,11 @@ void TmModuleOutputJsonRegister (void);
 void CreateJSONFlowId(json_t *js, const Flow *f);
 void JsonTcpFlags(uint8_t flags, json_t *js);
 json_t *CreateJSONHeader(Packet *p, int direction_sensative, char *event_type);
+json_t *CreateJSONHeaderWithTxId(Packet *p, int direction_sensitive, char *event_type, uint32_t tx_id);
 TmEcode OutputJSON(json_t *js, void *data, uint64_t *count);
 int OutputJSONBuffer(json_t *js, LogFileCtx *file_ctx, MemBuffer *buffer);
 OutputCtx *OutputJsonInitCtx(ConfNode *);
 
-
 enum JsonFormat { COMPACT, INDENT };
 
 /*
diff --git a/framework/src/suricata/src/output-lua.c b/framework/src/suricata/src/output-lua.c
index fc5a5621..a8fcb246 100644
--- a/framework/src/suricata/src/output-lua.c
+++ b/framework/src/suricata/src/output-lua.c
@@ -40,6 +40,8 @@
 #include "output.h"
 #include "app-layer-htp.h"
 #include "app-layer.h"
+#include "app-layer-ssl.h"
+#include "app-layer-ssh.h"
 #include "app-layer-parser.h"
 #include "util-privs.h"
 #include "util-buffer.h"
@@ -57,6 +59,8 @@
 #include "util-lua-common.h"
 #include "util-lua-http.h"
 #include "util-lua-dns.h"
+#include "util-lua-tls.h"
+#include "util-lua-ssh.h"
 
 #define MODULE_NAME "LuaLog"
 
@@ -229,6 +233,166 @@ static int LuaPacketConditionAlerts(ThreadVars *tv, const Packet *p)
 }
 
 /** \internal
+ *  \brief Packet Logger for lua scripts, for tls
+ *
+ *  A single call to this function will run one script for a single
+ *  packet. If it is called, it means that the registered condition
+ *  function has returned TRUE.
+ *
+ *  The script is called once for each packet.
+ */
+static int LuaPacketLoggerTls(ThreadVars *tv, void *thread_data, const Packet *p)
+{
+    LogLuaThreadCtx *td = (LogLuaThreadCtx *)thread_data;
+
+    char timebuf[64];
+    CreateTimeString(&p->ts, timebuf, sizeof(timebuf));
+
+    SCMutexLock(&td->lua_ctx->m);
+
+    lua_getglobal(td->lua_ctx->luastate, "log");
+
+    LuaStateSetThreadVars(td->lua_ctx->luastate, tv);
+    LuaStateSetPacket(td->lua_ctx->luastate, (Packet *)p);
+    LuaStateSetFlow(td->lua_ctx->luastate, p->flow, /* unlocked */LUA_FLOW_NOT_LOCKED_BY_PARENT);
+
+    int retval = lua_pcall(td->lua_ctx->luastate, 0, 0, 0);
+    if (retval != 0) {
+        SCLogInfo("failed to run script: %s", lua_tostring(td->lua_ctx->luastate, -1));
+    }
+
+    SCMutexUnlock(&td->lua_ctx->m);
+    FLOWLOCK_WRLOCK(p->flow);
+
+    SSLState *ssl_state = (SSLState *)FlowGetAppState(p->flow);
+    if (ssl_state != NULL)
+        ssl_state->flags |= SSL_AL_FLAG_STATE_LOGGED_LUA;
+
+    FLOWLOCK_UNLOCK(p->flow);
+    SCReturnInt(0);
+}
+
+static int LuaPacketConditionTls(ThreadVars *tv, const Packet *p)
+{
+    if (p->flow == NULL) {
+        return FALSE;
+    }
+
+    if (!(PKT_IS_IPV4(p)) && !(PKT_IS_IPV6(p))) {
+        return FALSE;
+    }
+
+    if (!(PKT_IS_TCP(p))) {
+        return FALSE;
+    }
+
+    FLOWLOCK_RDLOCK(p->flow);
+    uint16_t proto = FlowGetAppProtocol(p->flow);
+    if (proto != ALPROTO_TLS)
+        goto dontlog;
+
+    SSLState *ssl_state = (SSLState *)FlowGetAppState(p->flow);
+    if (ssl_state == NULL) {
+        SCLogDebug("no tls state, so no request logging");
+        goto dontlog;
+    }
+
+    if (ssl_state->server_connp.cert0_issuerdn == NULL ||
+            ssl_state->server_connp.cert0_subject == NULL)
+        goto dontlog;
+
+    /* We only log the state once */
+    if (ssl_state->flags & SSL_AL_FLAG_STATE_LOGGED_LUA)
+        goto dontlog;
+
+    FLOWLOCK_UNLOCK(p->flow);
+    return TRUE;
+dontlog:
+    FLOWLOCK_UNLOCK(p->flow);
+    return FALSE;
+}
+
+/** \internal
+ *  \brief Packet Logger for lua scripts, for ssh
+ *
+ *  A single call to this function will run one script for a single
+ *  packet. If it is called, it means that the registered condition
+ *  function has returned TRUE.
+ *
+ *  The script is called once for each packet.
+ */
+static int LuaPacketLoggerSsh(ThreadVars *tv, void *thread_data, const Packet *p)
+{
+    LogLuaThreadCtx *td = (LogLuaThreadCtx *)thread_data;
+
+    char timebuf[64];
+    CreateTimeString(&p->ts, timebuf, sizeof(timebuf));
+
+    SCMutexLock(&td->lua_ctx->m);
+
+    lua_getglobal(td->lua_ctx->luastate, "log");
+
+    LuaStateSetThreadVars(td->lua_ctx->luastate, tv);
+    LuaStateSetPacket(td->lua_ctx->luastate, (Packet *)p);
+    LuaStateSetFlow(td->lua_ctx->luastate, p->flow, /* unlocked */LUA_FLOW_NOT_LOCKED_BY_PARENT);
+
+    int retval = lua_pcall(td->lua_ctx->luastate, 0, 0, 0);
+    if (retval != 0) {
+        SCLogInfo("failed to run script: %s", lua_tostring(td->lua_ctx->luastate, -1));
+    }
+
+    SCMutexUnlock(&td->lua_ctx->m);
+    FLOWLOCK_WRLOCK(p->flow);
+
+    SshState *ssh_state = (SshState *)FlowGetAppState(p->flow);
+    if (ssh_state != NULL)
+        ssh_state->cli_hdr.flags |= SSH_FLAG_STATE_LOGGED_LUA;
+
+    FLOWLOCK_UNLOCK(p->flow);
+    SCReturnInt(0);
+}
+
+static int LuaPacketConditionSsh(ThreadVars *tv, const Packet *p)
+{
+    if (p->flow == NULL) {
+        return FALSE;
+    }
+
+    if (!(PKT_IS_IPV4(p)) && !(PKT_IS_IPV6(p))) {
+        return FALSE;
+    }
+
+    if (!(PKT_IS_TCP(p))) {
+        return FALSE;
+    }
+
+    FLOWLOCK_RDLOCK(p->flow);
+    uint16_t proto = FlowGetAppProtocol(p->flow);
+    if (proto != ALPROTO_SSH)
+        goto dontlog;
+
+    SshState *ssh_state = (SshState *)FlowGetAppState(p->flow);
+    if (ssh_state == NULL) {
+        SCLogDebug("no ssh state, so no request logging");
+        goto dontlog;
+    }
+
+    if (ssh_state->cli_hdr.software_version == NULL ||
+        ssh_state->srv_hdr.software_version == NULL)
+        goto dontlog;
+
+    /* We only log the state once */
+    if (ssh_state->cli_hdr.flags & SSH_FLAG_STATE_LOGGED_LUA)
+        goto dontlog;
+
+    FLOWLOCK_UNLOCK(p->flow);
+    return TRUE;
+dontlog:
+    FLOWLOCK_UNLOCK(p->flow);
+    return FALSE;
+}
+
+/** \internal
  *  \brief Packet Logger for lua scripts, for packets
  *
  *  A single call to this function will run one script for a single
@@ -517,6 +681,10 @@ static int LuaScriptInit(const char *filename, LogLuaScriptOptions *options) {
             options->alproto = ALPROTO_HTTP;
         else if (strcmp(k,"protocol") == 0 && strcmp(v, "dns") == 0)
             options->alproto = ALPROTO_DNS;
+        else if (strcmp(k,"protocol") == 0 && strcmp(v, "tls") == 0)
+            options->alproto = ALPROTO_TLS;
+        else if (strcmp(k,"protocol") == 0 && strcmp(v, "ssh") == 0)
+            options->alproto = ALPROTO_SSH;
         else if (strcmp(k, "type") == 0 && strcmp(v, "packet") == 0)
             options->packet = 1;
         else if (strcmp(k, "filter") == 0 && strcmp(v, "alerts") == 0)
@@ -617,6 +785,8 @@ static lua_State *LuaScriptSetup(const char *filename)
      * if the tx is registered in the state at runtime though. */
     LuaRegisterHttpFunctions(luastate);
     LuaRegisterDnsFunctions(luastate);
+    LuaRegisterTlsFunctions(luastate);
+    LuaRegisterSshFunctions(luastate);
 
     if (lua_pcall(luastate, 0, 0, 0) != 0) {
         SCLogError(SC_ERR_LUA_ERROR, "couldn't run script 'setup' function: %s", lua_tostring(luastate, -1));
@@ -760,11 +930,17 @@ static OutputCtx *OutputLuaLogInit(ConfNode *conf)
             om->TxLogFunc = LuaTxLogger;
             om->alproto = ALPROTO_HTTP;
             AppLayerParserRegisterLogger(IPPROTO_TCP, ALPROTO_HTTP);
+        } else if (opts.alproto == ALPROTO_TLS) {
+            om->PacketLogFunc = LuaPacketLoggerTls;
+            om->PacketConditionFunc = LuaPacketConditionTls;
         } else if (opts.alproto == ALPROTO_DNS) {
             om->TxLogFunc = LuaTxLogger;
             om->alproto = ALPROTO_DNS;
             AppLayerParserRegisterLogger(IPPROTO_TCP, ALPROTO_DNS);
             AppLayerParserRegisterLogger(IPPROTO_UDP, ALPROTO_DNS);
+        } else if (opts.alproto == ALPROTO_SSH) {
+            om->PacketLogFunc = LuaPacketLoggerSsh;
+            om->PacketConditionFunc = LuaPacketConditionSsh;
         } else if (opts.packet && opts.alerts) {
             om->PacketLogFunc = LuaPacketLoggerAlerts;
             om->PacketConditionFunc = LuaPacketConditionAlerts;
diff --git a/framework/src/suricata/src/runmode-pfring.c b/framework/src/suricata/src/runmode-pfring.c
index fbbd8c91..f08bdad8 100644
--- a/framework/src/suricata/src/runmode-pfring.c
+++ b/framework/src/suricata/src/runmode-pfring.c
@@ -109,6 +109,7 @@ void *OldParsePfringConfig(const char *iface)
     }
 
     strlcpy(pfconf->iface, iface, sizeof(pfconf->iface));
+    pfconf->flags = 0;
     pfconf->threads = 1;
     pfconf->cluster_id = 1;
 #ifdef HAVE_PFRING
@@ -143,6 +144,7 @@ void *OldParsePfringConfig(const char *iface)
         SCLogError(SC_ERR_INVALID_ARGUMENT,"Could not get cluster-id from config");
     } else {
         pfconf->cluster_id = (uint16_t)atoi(tmpclusterid);
+        pfconf->flags |= PFRING_CONF_FLAGS_CLUSTER;
         SCLogDebug("Going to use cluster-id %" PRId32, pfconf->cluster_id);
     }
 
@@ -263,6 +265,7 @@ void *ParsePfringConfig(const char *iface)
     /* command line value has precedence */
     if (ConfGet("pfring.cluster-id", &tmpclusterid) == 1) {
         pfconf->cluster_id = (uint16_t)atoi(tmpclusterid);
+        pfconf->flags |= PFRING_CONF_FLAGS_CLUSTER;
         SCLogDebug("Going to use command-line provided cluster-id %" PRId32,
                    pfconf->cluster_id);
     } else {
@@ -278,6 +281,7 @@ void *ParsePfringConfig(const char *iface)
                        "Could not get cluster-id from config");
         } else {
             pfconf->cluster_id = (uint16_t)atoi(tmpclusterid);
+            pfconf->flags |= PFRING_CONF_FLAGS_CLUSTER;
             SCLogDebug("Going to use cluster-id %" PRId32, pfconf->cluster_id);
         }
     }
diff --git a/framework/src/suricata/src/runmode-unittests.c b/framework/src/suricata/src/runmode-unittests.c
index 4f3fc8bf..3701d29a 100644
--- a/framework/src/suricata/src/runmode-unittests.c
+++ b/framework/src/suricata/src/runmode-unittests.c
@@ -277,6 +277,7 @@ void RunUnittests(int list_unittests, char *regex_arg)
     CudaBufferRegisterUnittests();
 #endif
     AppLayerUnittestsRegister();
+    MimeDecRegisterTests();
     if (list_unittests) {
         UtListTests(regex_arg);
     } else {
diff --git a/framework/src/suricata/src/source-pfring.c b/framework/src/suricata/src/source-pfring.c
index 3b2b52df..527086f5 100644
--- a/framework/src/suricata/src/source-pfring.c
+++ b/framework/src/suricata/src/source-pfring.c
@@ -125,6 +125,10 @@ static SCMutex pfring_bpf_set_filter_lock = SCMUTEX_INITIALIZER;
 #define LIBPFRING_REENTRANT   0
 #define LIBPFRING_WAIT_FOR_INCOMING 1
 
+typedef enum {
+    PFRING_FLAGS_ZERO_COPY = 0x1
+} PfringThreadVarsFlags;
+
 /**
  * \brief Structure to hold thread specific variables.
  */
@@ -140,6 +144,8 @@ typedef struct PfringThreadVars_
     uint16_t capture_kernel_packets;
     uint16_t capture_kernel_drops;
 
+    uint32_t flags;
+
     ThreadVars *tv;
     TmSlot *slot;
 
@@ -295,7 +301,8 @@ TmEcode ReceivePfringLoop(ThreadVars *tv, void *data, void *slot)
     struct pfring_pkthdr hdr;
     TmSlot *s = (TmSlot *)slot;
     time_t last_dump = 0;
-    struct timeval current_time;
+    u_int buffer_size;
+    u_char *pkt_buffer;
 
     ptv->slot = s->slot_next;
 
@@ -325,16 +332,23 @@ TmEcode ReceivePfringLoop(ThreadVars *tv, void *data, void *slot)
         /* Some flavours of PF_RING may fail to set timestamp - see PF-RING-enabled libpcap code*/
         hdr.ts.tv_sec = hdr.ts.tv_usec = 0;
 
+        /* Check for Zero-copy mode */
+        if (ptv->flags & PFRING_FLAGS_ZERO_COPY) {
+            buffer_size = 0;
+            pkt_buffer = NULL;
+        } else {
+            buffer_size = GET_PKT_DIRECT_MAX_SIZE(p);
+            pkt_buffer = GET_PKT_DIRECT_DATA(p);
+        }
+
         /* Depending on what compile time options are used for pfring we either return 0 or -1 on error and always 1 for success */
-        u_char *pkt_buffer = GET_PKT_DIRECT_DATA(p);
-        u_int buffer_size = GET_PKT_DIRECT_MAX_SIZE(p);
         int r = pfring_recv(ptv->pd, &pkt_buffer,
                 buffer_size,
                 &hdr,
                 LIBPFRING_WAIT_FOR_INCOMING);
 
-        /* Check for Zero-copy if buffer size is zero */
-        if (buffer_size == 0) {
+        /* Check for Zero-copy mode */
+        if (ptv->flags & PFRING_FLAGS_ZERO_COPY) {
             PacketSetData(p, pkt_buffer, hdr.caplen);
         }
 
@@ -350,10 +364,9 @@ TmEcode ReceivePfringLoop(ThreadVars *tv, void *data, void *slot)
             }
 
             /* Trigger one dump of stats every second */
-            TimeGet(&current_time);
-            if (current_time.tv_sec != last_dump) {
+            if (p->ts.tv_sec != last_dump) {
                 PfringDumpCounters(ptv);
-                last_dump = current_time.tv_sec;
+                last_dump = p->ts.tv_sec;
             }
         } else {
             SCLogError(SC_ERR_PF_RING_RECV,"pfring_recv error  %" PRId32 "", r);
@@ -386,7 +399,7 @@ TmEcode ReceivePfringThreadInit(ThreadVars *tv, void *initdata, void **data)
     u_int32_t version = 0;
     PfringIfaceConfig *pfconf = (PfringIfaceConfig *) initdata;
     unsigned int opflag;
-
+    char const *active_runmode = RunmodeGetActive();
 
     if (pfconf == NULL)
         return TM_ECODE_FAILED;
@@ -415,9 +428,15 @@ TmEcode ReceivePfringThreadInit(ThreadVars *tv, void *initdata, void **data)
         SCReturnInt(TM_ECODE_FAILED);
     }
 
+    /* enable zero-copy mode for workers runmode */
+    if (active_runmode && strcmp("workers", active_runmode) == 0) {
+        ptv->flags |= PFRING_FLAGS_ZERO_COPY;
+        SCLogInfo("Enabling zero-copy for %s", ptv->interface);
+    }
+
     ptv->checksum_mode = pfconf->checksum_mode;
 
-    opflag = PF_RING_REENTRANT | PF_RING_PROMISC;
+    opflag = PF_RING_PROMISC;
 
     /* if suri uses VLAN and if we have a recent kernel, we need
      * to use parsed_pkt to get VLAN info */
@@ -466,8 +485,11 @@ TmEcode ReceivePfringThreadInit(ThreadVars *tv, void *initdata, void **data)
         if (rc != 0) {
             SCLogError(SC_ERR_PF_RING_SET_CLUSTER_FAILED, "pfring_set_cluster "
                     "returned %d for cluster-id: %d", rc, ptv->cluster_id);
-            pfconf->DerefFunc(pfconf);
-            return TM_ECODE_FAILED;
+            if (rc != PF_RING_ERROR_NOT_SUPPORTED || (pfconf->flags & PFRING_CONF_FLAGS_CLUSTER)) {
+                /* cluster is mandatory as explicitly specified in the configuration */
+                pfconf->DerefFunc(pfconf);
+                return TM_ECODE_FAILED;
+            }
         }
     }
 
diff --git a/framework/src/suricata/src/source-pfring.h b/framework/src/suricata/src/source-pfring.h
index e0878454..9871f458 100644
--- a/framework/src/suricata/src/source-pfring.h
+++ b/framework/src/suricata/src/source-pfring.h
@@ -31,8 +31,14 @@
 #include <pfring.h>
 #endif
 
+typedef enum {
+    PFRING_CONF_FLAGS_CLUSTER = 0x1
+} PfringIfaceConfigFlags;
+
 typedef struct PfringIfaceConfig_
 {
+    uint32_t flags;
+
     /* cluster param */
     int cluster_id;
 #ifdef HAVE_PFRING
diff --git a/framework/src/suricata/src/stream-tcp-reassemble.c b/framework/src/suricata/src/stream-tcp-reassemble.c
index caf955af..a947fab3 100644
--- a/framework/src/suricata/src/stream-tcp-reassemble.c
+++ b/framework/src/suricata/src/stream-tcp-reassemble.c
@@ -2620,51 +2620,58 @@ static inline int DoReassemble(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
                  TcpSession *ssn, TcpStream *stream, TcpSegment *seg, ReassembleData *rd,
                  Packet *p)
 {
-    /* fast path 1: segment is exactly what we need */
-    if (likely(rd->data_len == 0 &&
-               SEQ_EQ(seg->seq, rd->ra_base_seq+1) &&
-               SEQ_EQ(stream->last_ack, (seg->seq + seg->payload_len))))
-    {
-        /* process single segment directly */
-        AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream,
-                seg->payload, seg->payload_len,
-                StreamGetAppLayerFlags(ssn, stream, p));
-        AppLayerProfilingStore(ra_ctx->app_tctx, p);
-        rd->data_sent += seg->payload_len;
-        rd->ra_base_seq += seg->payload_len;
+    /* fast paths: send data directly into the app layer, w/o first doing
+     * a copy step. However, don't use the fast path until protocol detection
+     * has been completed
+     * TODO if initial data is big enough for proto detect, we could do the
+     *      fast path anyway. */
+    if (stream->flags & STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_COMPLETED) {
+        /* fast path 1: segment is exactly what we need */
+        if (likely(rd->data_len == 0 &&
+                    SEQ_EQ(seg->seq, rd->ra_base_seq+1) &&
+                    SEQ_EQ(stream->last_ack, (seg->seq + seg->payload_len))))
+        {
+            /* process single segment directly */
+            AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream,
+                    seg->payload, seg->payload_len,
+                    StreamGetAppLayerFlags(ssn, stream, p));
+            AppLayerProfilingStore(ra_ctx->app_tctx, p);
+            rd->data_sent += seg->payload_len;
+            rd->ra_base_seq += seg->payload_len;
 #ifdef DEBUG
-        ra_ctx->fp1++;
+            ra_ctx->fp1++;
 #endif
-        /* if after the first data chunk we have no alproto yet,
-         * there is no point in continueing here. */
-        if (!StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(stream)) {
-            SCLogDebug("no alproto after first data chunk");
-            return 0;
-        }
-        return 1;
-    /* fast path 2: segment acked completely, meets minimal size req for 0copy processing */
-    } else if (rd->data_len == 0 &&
-               SEQ_EQ(seg->seq, rd->ra_base_seq+1) &&
-               SEQ_GT(stream->last_ack, (seg->seq + seg->payload_len)) &&
-               seg->payload_len >= stream_config.zero_copy_size)
-    {
-        /* process single segment directly */
-        AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream,
-                seg->payload, seg->payload_len,
-                StreamGetAppLayerFlags(ssn, stream, p));
-        AppLayerProfilingStore(ra_ctx->app_tctx, p);
-        rd->data_sent += seg->payload_len;
-        rd->ra_base_seq += seg->payload_len;
+            /* if after the first data chunk we have no alproto yet,
+             * there is no point in continueing here. */
+            if (!StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(stream)) {
+                SCLogDebug("no alproto after first data chunk");
+                return 0;
+            }
+            return 1;
+            /* fast path 2: segment acked completely, meets minimal size req for 0copy processing */
+        } else if (rd->data_len == 0 &&
+                SEQ_EQ(seg->seq, rd->ra_base_seq+1) &&
+                SEQ_GT(stream->last_ack, (seg->seq + seg->payload_len)) &&
+                seg->payload_len >= stream_config.zero_copy_size)
+        {
+            /* process single segment directly */
+            AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream,
+                    seg->payload, seg->payload_len,
+                    StreamGetAppLayerFlags(ssn, stream, p));
+            AppLayerProfilingStore(ra_ctx->app_tctx, p);
+            rd->data_sent += seg->payload_len;
+            rd->ra_base_seq += seg->payload_len;
 #ifdef DEBUG
-        ra_ctx->fp2++;
+            ra_ctx->fp2++;
 #endif
-        /* if after the first data chunk we have no alproto yet,
-         * there is no point in continueing here. */
-        if (!StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(stream)) {
-            SCLogDebug("no alproto after first data chunk");
-            return 0;
+            /* if after the first data chunk we have no alproto yet,
+             * there is no point in continueing here. */
+            if (!StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(stream)) {
+                SCLogDebug("no alproto after first data chunk");
+                return 0;
+            }
+            return 1;
         }
-        return 1;
     }
 #ifdef DEBUG
     ra_ctx->sp++;
@@ -2888,6 +2895,49 @@ int StreamTcpReassembleAppLayer (ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
     GetSessionSize(ssn, p);
 #endif
 
+    /* Check if we have a gap at the start of the stream. 2 conditions:
+     * 1. no segments, but last_ack moved fwd
+     * 2. segments, but clearly some missing: if last_ack is
+     *    bigger than the list start and the list start is bigger than
+     *    next_seq, we know we are missing data that has been ack'd. That
+     *    won't get retransmitted, so it's a data gap.
+     */
+    if (!(ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED)) {
+        int ackadd = (ssn->state >= TCP_FIN_WAIT2) ? 2 : 1;
+        if ((stream->seg_list == NULL && /*1*/
+                    stream->ra_app_base_seq == stream->isn &&
+                    SEQ_GT(stream->last_ack, stream->isn + ackadd))
+                ||
+            (stream->seg_list != NULL && /*2*/
+                    SEQ_GT(stream->seg_list->seq, stream->ra_app_base_seq+1) &&
+                    SEQ_LT(stream->seg_list->seq, stream->last_ack)))
+        {
+            if (stream->seg_list == NULL) {
+                SCLogDebug("no segs, last_ack moved fwd so GAP "
+                        "(base %u, isn %u, last_ack %u => diff %u) p %"PRIu64,
+                        stream->ra_app_base_seq, stream->isn, stream->last_ack,
+                        stream->last_ack - (stream->isn + ackadd), p->pcap_cnt);
+            }
+
+            /* send gap signal */
+            AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream,
+                    NULL, 0,
+                    StreamGetAppLayerFlags(ssn, stream, p)|STREAM_GAP);
+            AppLayerProfilingStore(ra_ctx->app_tctx, p);
+
+            /* set a GAP flag and make sure not bothering this stream anymore */
+            SCLogDebug("STREAMTCP_STREAM_FLAG_GAP set");
+            stream->flags |= STREAMTCP_STREAM_FLAG_GAP;
+
+            StreamTcpSetEvent(p, STREAM_REASSEMBLY_SEQ_GAP);
+            StatsIncr(tv, ra_ctx->counter_tcp_reass_gap);
+#ifdef DEBUG
+            dbg_app_layer_gap++;
+#endif
+            SCReturnInt(0);
+        }
+    }
+
     /* if no segments are in the list or all are already processed,
      * and state is beyond established, we send an empty msg */
     TcpSegment *seg_tail = stream->seg_list_tail;
@@ -2934,35 +2984,14 @@ int StreamTcpReassembleAppLayer (ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
     /* loop through the segments and fill one or more msgs */
     TcpSegment *seg = stream->seg_list;
     SCLogDebug("pre-loop seg %p", seg);
-
-    /* Check if we have a gap at the start of the list. If last_ack is
-     * bigger than the list start and the list start is bigger than
-     * next_seq, we know we are missing data that has been ack'd. That
-     * won't get retransmitted, so it's a data gap.
-     */
-    if (!(ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED)) {
-        if (SEQ_GT(seg->seq, next_seq) && SEQ_LT(seg->seq, stream->last_ack)) {
-            /* send gap signal */
-            AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream,
-                    NULL, 0,
-                    StreamGetAppLayerFlags(ssn, stream, p)|STREAM_GAP);
-            AppLayerProfilingStore(ra_ctx->app_tctx, p);
-
-            /* set a GAP flag and make sure not bothering this stream anymore */
-            SCLogDebug("STREAMTCP_STREAM_FLAG_GAP set");
-            stream->flags |= STREAMTCP_STREAM_FLAG_GAP;
-
-            StreamTcpSetEvent(p, STREAM_REASSEMBLY_SEQ_GAP);
-            StatsIncr(tv, ra_ctx->counter_tcp_reass_gap);
-#ifdef DEBUG
-            dbg_app_layer_gap++;
+#ifdef DEBUG_VALIDATION
+    uint64_t bytes = 0;
 #endif
-            SCReturnInt(0);
-        }
-    }
-
     for (; seg != NULL; )
     {
+#ifdef DEBUG_VALIDATION
+        bytes += seg->payload_len;
+#endif
         /* if in inline mode, we process all segments regardless of whether
          * they are ack'd or not. In non-inline, we process only those that
          * are at least partly ack'd. */
@@ -3007,6 +3036,9 @@ int StreamTcpReassembleAppLayer (ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
         }
         seg = next_seg;
     }
+#ifdef DEBUG_VALIDATION /* we should never have this much data queued */
+    BUG_ON(bytes > 1000000ULL && bytes > (stream->window * 1.5));
+#endif
 
     /* put the partly filled smsg in the queue to the l7 handler */
     if (rd.data_len > 0) {
@@ -3035,6 +3067,8 @@ int StreamTcpReassembleAppLayer (ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
     } else {
         TcpSegment *tmp_seg = stream->seg_list;
         while (tmp_seg != NULL) {
+            if (!(tmp_seg->flags & SEGMENTTCP_FLAG_APPLAYER_PROCESSED))
+                break;
             tmp_seg->flags &= ~SEGMENTTCP_FLAG_APPLAYER_PROCESSED;
             tmp_seg = tmp_seg->next;
         }
@@ -5417,6 +5451,9 @@ static int StreamTcpReassembleTest30 (void)
     th_flag = TH_ACK|TH_PUSH;
     th_flags = TH_ACK;
 
+    ssn.client.last_ack = 2;
+    ssn.client.isn = 1;
+
     ssn.server.last_ack = 22;
     ssn.server.ra_raw_base_seq = ssn.server.ra_app_base_seq = 9;
     ssn.server.isn = 9;
@@ -6100,7 +6137,7 @@ static int StreamTcpReassembleTest38 (void)
     ssn.server.last_ack = 60;
     ssn.client.ra_raw_base_seq = ssn.client.ra_app_base_seq = 9;
     ssn.client.isn = 9;
-    ssn.client.last_ack = 60;
+    ssn.client.last_ack = 9;
     f.alproto = ALPROTO_UNKNOWN;
 
     f.flags |= FLOW_IPV4;
@@ -7218,7 +7255,7 @@ static int StreamTcpReassembleTest45 (void)
     ssn.server.last_ack = 60;
     STREAMTCP_SET_RA_BASE_SEQ(&ssn.client, 9);
     ssn.client.isn = 9;
-    ssn.client.last_ack = 60;
+    ssn.client.last_ack = 9;
 
     f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 200, 220);
     if (f == NULL)
@@ -7337,7 +7374,7 @@ static int StreamTcpReassembleTest46 (void)
     ssn.server.next_seq = ssn.server.isn;
     STREAMTCP_SET_RA_BASE_SEQ(&ssn.client, 9);
     ssn.client.isn = 9;
-    ssn.client.last_ack = 60;
+    ssn.client.last_ack = 9;
     ssn.client.next_seq = ssn.client.isn;
 
     f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 200, 220);
@@ -8405,6 +8442,9 @@ static int StreamTcpReassembleInlineTest10(void)
     StreamTcpUTInitInline();
     StreamTcpUTSetupSession(&ssn);
     StreamTcpUTSetupStream(&ssn.server, 1);
+    ssn.server.last_ack = 2;
+    StreamTcpUTSetupStream(&ssn.client, 1);
+    ssn.client.last_ack = 2;
 
     f = UTHBuildFlow(AF_INET, "1.1.1.1", "2.2.2.2", 1024, 80);
     if (f == NULL)
diff --git a/framework/src/suricata/src/stream-tcp.c b/framework/src/suricata/src/stream-tcp.c
index 6cde8651..9dce7070 100644
--- a/framework/src/suricata/src/stream-tcp.c
+++ b/framework/src/suricata/src/stream-tcp.c
@@ -764,8 +764,13 @@ uint32_t StreamTcpGetStreamSize(TcpStream *stream)
 #define StreamTcpUpdateLastAck(ssn, stream, ack) { \
     if (SEQ_GT((ack), (stream)->last_ack)) \
     { \
+        SCLogDebug("ssn %p: last_ack set to %"PRIu32", moved %u forward", (ssn), (ack), (ack) - (stream)->last_ack); \
+        if ((SEQ_LEQ((stream)->last_ack, (stream)->next_seq) && SEQ_GT((ack),(stream)->next_seq))) { \
+            SCLogDebug("last_ack just passed next_seq: %u (was %u) > %u", (ack), (stream)->last_ack, (stream)->next_seq); \
+        } else { \
+            SCLogDebug("next_seq (%u) <> last_ack now %d", (stream)->next_seq, (int)(stream)->next_seq - (ack)); \
+        }\
         (stream)->last_ack = (ack); \
-        SCLogDebug("ssn %p: last_ack set to %"PRIu32, (ssn), (stream)->last_ack); \
         StreamTcpSackPruneList((stream)); \
     } else { \
         SCLogDebug("ssn %p: no update: ack %u, last_ack %"PRIu32", next_seq %u (state %u)", \
@@ -793,6 +798,13 @@ static int StreamTcpPacketIsRetransmission(TcpStream *stream, Packet *p)
     if (p->payload_len == 0)
         SCReturnInt(0);
 
+    /* retransmission of already partially ack'd data */
+    if (SEQ_LT(TCP_GET_SEQ(p), stream->last_ack) && SEQ_GT((TCP_GET_SEQ(p) + p->payload_len), stream->last_ack))
+    {
+        StreamTcpSetEvent(p, STREAM_PKT_RETRANSMISSION);
+        SCReturnInt(1);
+    }
+
     /* retransmission of already ack'd data */
     if (SEQ_LEQ((TCP_GET_SEQ(p) + p->payload_len), stream->last_ack)) {
         StreamTcpSetEvent(p, STREAM_PKT_RETRANSMISSION);
@@ -805,8 +817,8 @@ static int StreamTcpPacketIsRetransmission(TcpStream *stream, Packet *p)
         SCReturnInt(2);
     }
 
-    SCLogDebug("seq %u payload_len %u => %u, last_ack %u", TCP_GET_SEQ(p),
-            p->payload_len, (TCP_GET_SEQ(p) + p->payload_len), stream->last_ack);
+    SCLogDebug("seq %u payload_len %u => %u, last_ack %u, next_seq %u", TCP_GET_SEQ(p),
+            p->payload_len, (TCP_GET_SEQ(p) + p->payload_len), stream->last_ack, stream->next_seq);
     SCReturnInt(0);
 }
 
@@ -2040,6 +2052,17 @@ static int HandleEstablishedPacketToServer(ThreadVars *tv, TcpSession *ssn, Pack
         SCLogDebug("ssn %p: ssn->client.next_seq %"PRIu32
                    " (started before next_seq, ended after)",
                    ssn, ssn->client.next_seq);
+    /* if next_seq has fallen behind last_ack, we got some catching up to do */
+    } else if (SEQ_LT(ssn->client.next_seq, ssn->client.last_ack)) {
+        ssn->client.next_seq = TCP_GET_SEQ(p) + p->payload_len;
+        SCLogDebug("ssn %p: ssn->client.next_seq %"PRIu32
+                   " (next_seq had fallen behind last_ack)",
+                   ssn, ssn->client.next_seq);
+    } else {
+        SCLogDebug("ssn %p: no update to ssn->client.next_seq %"PRIu32
+                   " SEQ %u SEQ+ %u last_ack %u",
+                   ssn, ssn->client.next_seq,
+                   TCP_GET_SEQ(p), TCP_GET_SEQ(p)+p->payload_len, ssn->client.last_ack);
     }
 
     /* in window check */
@@ -2185,6 +2208,17 @@ static int HandleEstablishedPacketToClient(ThreadVars *tv, TcpSession *ssn, Pack
         SCLogDebug("ssn %p: ssn->server.next_seq %" PRIu32
                    " (started before next_seq, ended after)",
                    ssn, ssn->server.next_seq);
+    /* if next_seq has fallen behind last_ack, we got some catching up to do */
+    } else if (SEQ_LT(ssn->server.next_seq, ssn->server.last_ack)) {
+        ssn->server.next_seq = TCP_GET_SEQ(p) + p->payload_len;
+        SCLogDebug("ssn %p: ssn->server.next_seq %"PRIu32
+                   " (next_seq had fallen behind last_ack)",
+                   ssn, ssn->server.next_seq);
+    } else {
+        SCLogDebug("ssn %p: no update to ssn->server.next_seq %"PRIu32
+                   " SEQ %u SEQ+ %u last_ack %u",
+                   ssn, ssn->server.next_seq,
+                   TCP_GET_SEQ(p), TCP_GET_SEQ(p)+p->payload_len, ssn->server.last_ack);
     }
 
     if (zerowindowprobe) {
@@ -2284,8 +2318,9 @@ static int StreamTcpPacketStateEstablished(ThreadVars *tv, Packet *p,
                     ssn->server.next_seq);
             ssn->client.window = TCP_GET_WINDOW(p) << ssn->client.wscale;
 
-            StreamTcpUpdateLastAck(ssn, &ssn->server,
-                    StreamTcpResetGetMaxAck(&ssn->server, TCP_GET_ACK(p)));
+            if ((p->tcph->th_flags & TH_ACK) && StreamTcpValidateAck(ssn, &ssn->server, p) == 0)
+                StreamTcpUpdateLastAck(ssn, &ssn->server,
+                        StreamTcpResetGetMaxAck(&ssn->server, TCP_GET_ACK(p)));
 
             StreamTcpUpdateLastAck(ssn, &ssn->client,
                     StreamTcpResetGetMaxAck(&ssn->client, TCP_GET_SEQ(p)));
@@ -2315,8 +2350,9 @@ static int StreamTcpPacketStateEstablished(ThreadVars *tv, Packet *p,
                     ssn->server.next_seq);
             ssn->server.window = TCP_GET_WINDOW(p) << ssn->server.wscale;
 
-            StreamTcpUpdateLastAck(ssn, &ssn->client,
-                    StreamTcpResetGetMaxAck(&ssn->client, TCP_GET_ACK(p)));
+            if ((p->tcph->th_flags & TH_ACK) && StreamTcpValidateAck(ssn, &ssn->client, p) == 0)
+                StreamTcpUpdateLastAck(ssn, &ssn->client,
+                        StreamTcpResetGetMaxAck(&ssn->client, TCP_GET_ACK(p)));
 
             StreamTcpUpdateLastAck(ssn, &ssn->server,
                     StreamTcpResetGetMaxAck(&ssn->server, TCP_GET_SEQ(p)));
@@ -2609,8 +2645,9 @@ static int StreamTcpPacketStateFinWait1(ThreadVars *tv, Packet *p,
                 ssn);
 
         if (PKT_IS_TOSERVER(p)) {
-            StreamTcpUpdateLastAck(ssn, &ssn->server,
-                    StreamTcpResetGetMaxAck(&ssn->server, TCP_GET_ACK(p)));
+            if ((p->tcph->th_flags & TH_ACK) && StreamTcpValidateAck(ssn, &ssn->server, p) == 0)
+                StreamTcpUpdateLastAck(ssn, &ssn->server,
+                        StreamTcpResetGetMaxAck(&ssn->server, TCP_GET_ACK(p)));
 
             StreamTcpUpdateLastAck(ssn, &ssn->client,
                     StreamTcpResetGetMaxAck(&ssn->client, TCP_GET_SEQ(p)));
@@ -2622,8 +2659,9 @@ static int StreamTcpPacketStateFinWait1(ThreadVars *tv, Packet *p,
             StreamTcpReassembleHandleSegment(tv, stt->ra_ctx, ssn,
                     &ssn->client, p, pq);
         } else {
-            StreamTcpUpdateLastAck(ssn, &ssn->client,
-                    StreamTcpResetGetMaxAck(&ssn->client, TCP_GET_ACK(p)));
+            if ((p->tcph->th_flags & TH_ACK) && StreamTcpValidateAck(ssn, &ssn->client, p) == 0)
+                StreamTcpUpdateLastAck(ssn, &ssn->client,
+                        StreamTcpResetGetMaxAck(&ssn->client, TCP_GET_ACK(p)));
 
             StreamTcpUpdateLastAck(ssn, &ssn->server,
                     StreamTcpResetGetMaxAck(&ssn->server, TCP_GET_SEQ(p)));
@@ -3063,8 +3101,9 @@ static int StreamTcpPacketStateFinWait2(ThreadVars *tv, Packet *p,
                 ssn);
 
         if (PKT_IS_TOSERVER(p)) {
-            StreamTcpUpdateLastAck(ssn, &ssn->server,
-                    StreamTcpResetGetMaxAck(&ssn->server, TCP_GET_ACK(p)));
+            if ((p->tcph->th_flags & TH_ACK) && StreamTcpValidateAck(ssn, &ssn->server, p) == 0)
+                StreamTcpUpdateLastAck(ssn, &ssn->server,
+                        StreamTcpResetGetMaxAck(&ssn->server, TCP_GET_ACK(p)));
 
             StreamTcpUpdateLastAck(ssn, &ssn->client,
                     StreamTcpResetGetMaxAck(&ssn->client, TCP_GET_SEQ(p)));
@@ -3076,8 +3115,9 @@ static int StreamTcpPacketStateFinWait2(ThreadVars *tv, Packet *p,
             StreamTcpReassembleHandleSegment(tv, stt->ra_ctx, ssn,
                     &ssn->client, p, pq);
         } else {
-            StreamTcpUpdateLastAck(ssn, &ssn->client,
-                    StreamTcpResetGetMaxAck(&ssn->client, TCP_GET_ACK(p)));
+            if ((p->tcph->th_flags & TH_ACK) && StreamTcpValidateAck(ssn, &ssn->client, p) == 0)
+                StreamTcpUpdateLastAck(ssn, &ssn->client,
+                        StreamTcpResetGetMaxAck(&ssn->client, TCP_GET_ACK(p)));
 
             StreamTcpUpdateLastAck(ssn, &ssn->server,
                     StreamTcpResetGetMaxAck(&ssn->server, TCP_GET_SEQ(p)));
@@ -3369,8 +3409,9 @@ static int StreamTcpPacketStateClosing(ThreadVars *tv, Packet *p,
                 ssn);
 
         if (PKT_IS_TOSERVER(p)) {
-            StreamTcpUpdateLastAck(ssn, &ssn->server,
-                    StreamTcpResetGetMaxAck(&ssn->server, TCP_GET_ACK(p)));
+            if ((p->tcph->th_flags & TH_ACK) && StreamTcpValidateAck(ssn, &ssn->server, p) == 0)
+                StreamTcpUpdateLastAck(ssn, &ssn->server,
+                        StreamTcpResetGetMaxAck(&ssn->server, TCP_GET_ACK(p)));
 
             StreamTcpUpdateLastAck(ssn, &ssn->client,
                     StreamTcpResetGetMaxAck(&ssn->client, TCP_GET_SEQ(p)));
@@ -3382,8 +3423,9 @@ static int StreamTcpPacketStateClosing(ThreadVars *tv, Packet *p,
             StreamTcpReassembleHandleSegment(tv, stt->ra_ctx, ssn,
                     &ssn->client, p, pq);
         } else {
-            StreamTcpUpdateLastAck(ssn, &ssn->client,
-                    StreamTcpResetGetMaxAck(&ssn->client, TCP_GET_ACK(p)));
+            if ((p->tcph->th_flags & TH_ACK) && StreamTcpValidateAck(ssn, &ssn->client, p) == 0)
+                StreamTcpUpdateLastAck(ssn, &ssn->client,
+                        StreamTcpResetGetMaxAck(&ssn->client, TCP_GET_ACK(p)));
 
             StreamTcpUpdateLastAck(ssn, &ssn->server,
                     StreamTcpResetGetMaxAck(&ssn->server, TCP_GET_SEQ(p)));
@@ -3549,8 +3591,9 @@ static int StreamTcpPacketStateCloseWait(ThreadVars *tv, Packet *p,
                 ssn);
 
         if (PKT_IS_TOSERVER(p)) {
-            StreamTcpUpdateLastAck(ssn, &ssn->server,
-                    StreamTcpResetGetMaxAck(&ssn->server, TCP_GET_ACK(p)));
+            if ((p->tcph->th_flags & TH_ACK) && StreamTcpValidateAck(ssn, &ssn->server, p) == 0)
+                StreamTcpUpdateLastAck(ssn, &ssn->server,
+                        StreamTcpResetGetMaxAck(&ssn->server, TCP_GET_ACK(p)));
 
             StreamTcpUpdateLastAck(ssn, &ssn->client,
                     StreamTcpResetGetMaxAck(&ssn->client, TCP_GET_SEQ(p)));
@@ -3562,8 +3605,9 @@ static int StreamTcpPacketStateCloseWait(ThreadVars *tv, Packet *p,
             StreamTcpReassembleHandleSegment(tv, stt->ra_ctx, ssn,
                     &ssn->client, p, pq);
         } else {
-            StreamTcpUpdateLastAck(ssn, &ssn->client,
-                    StreamTcpResetGetMaxAck(&ssn->client, TCP_GET_ACK(p)));
+            if ((p->tcph->th_flags & TH_ACK) && StreamTcpValidateAck(ssn, &ssn->client, p) == 0)
+                StreamTcpUpdateLastAck(ssn, &ssn->client,
+                        StreamTcpResetGetMaxAck(&ssn->client, TCP_GET_ACK(p)));
 
             StreamTcpUpdateLastAck(ssn, &ssn->server,
                     StreamTcpResetGetMaxAck(&ssn->server, TCP_GET_SEQ(p)));
@@ -3840,8 +3884,9 @@ static int StreamTcpPacketStateLastAck(ThreadVars *tv, Packet *p,
                 ssn);
 
         if (PKT_IS_TOSERVER(p)) {
-            StreamTcpUpdateLastAck(ssn, &ssn->server,
-                    StreamTcpResetGetMaxAck(&ssn->server, TCP_GET_ACK(p)));
+            if ((p->tcph->th_flags & TH_ACK) && StreamTcpValidateAck(ssn, &ssn->server, p) == 0)
+                StreamTcpUpdateLastAck(ssn, &ssn->server,
+                        StreamTcpResetGetMaxAck(&ssn->server, TCP_GET_ACK(p)));
 
             StreamTcpUpdateLastAck(ssn, &ssn->client,
                     StreamTcpResetGetMaxAck(&ssn->client, TCP_GET_SEQ(p)));
@@ -3853,8 +3898,9 @@ static int StreamTcpPacketStateLastAck(ThreadVars *tv, Packet *p,
             StreamTcpReassembleHandleSegment(tv, stt->ra_ctx, ssn,
                     &ssn->client, p, pq);
         } else {
-            StreamTcpUpdateLastAck(ssn, &ssn->client,
-                    StreamTcpResetGetMaxAck(&ssn->client, TCP_GET_ACK(p)));
+            if ((p->tcph->th_flags & TH_ACK) && StreamTcpValidateAck(ssn, &ssn->client, p) == 0)
+                StreamTcpUpdateLastAck(ssn, &ssn->client,
+                        StreamTcpResetGetMaxAck(&ssn->client, TCP_GET_ACK(p)));
 
             StreamTcpUpdateLastAck(ssn, &ssn->server,
                     StreamTcpResetGetMaxAck(&ssn->server, TCP_GET_SEQ(p)));
@@ -3965,8 +4011,9 @@ static int StreamTcpPacketStateTimeWait(ThreadVars *tv, Packet *p,
                 ssn);
 
         if (PKT_IS_TOSERVER(p)) {
-            StreamTcpUpdateLastAck(ssn, &ssn->server,
-                    StreamTcpResetGetMaxAck(&ssn->server, TCP_GET_ACK(p)));
+            if ((p->tcph->th_flags & TH_ACK) && StreamTcpValidateAck(ssn, &ssn->server, p) == 0)
+                StreamTcpUpdateLastAck(ssn, &ssn->server,
+                        StreamTcpResetGetMaxAck(&ssn->server, TCP_GET_ACK(p)));
 
             StreamTcpUpdateLastAck(ssn, &ssn->client,
                     StreamTcpResetGetMaxAck(&ssn->client, TCP_GET_SEQ(p)));
@@ -3978,8 +4025,9 @@ static int StreamTcpPacketStateTimeWait(ThreadVars *tv, Packet *p,
             StreamTcpReassembleHandleSegment(tv, stt->ra_ctx, ssn,
                     &ssn->client, p, pq);
         } else {
-            StreamTcpUpdateLastAck(ssn, &ssn->client,
-                    StreamTcpResetGetMaxAck(&ssn->client, TCP_GET_ACK(p)));
+            if ((p->tcph->th_flags & TH_ACK) && StreamTcpValidateAck(ssn, &ssn->client, p) == 0)
+                StreamTcpUpdateLastAck(ssn, &ssn->client,
+                        StreamTcpResetGetMaxAck(&ssn->client, TCP_GET_ACK(p)));
 
             StreamTcpUpdateLastAck(ssn, &ssn->server,
                     StreamTcpResetGetMaxAck(&ssn->server, TCP_GET_SEQ(p)));
diff --git a/framework/src/suricata/src/suricata.c b/framework/src/suricata/src/suricata.c
index 6c45c57e..173ef75f 100644
--- a/framework/src/suricata/src/suricata.c
+++ b/framework/src/suricata/src/suricata.c
@@ -1977,14 +1977,20 @@ static int ConfigGetCaptureValue(SCInstance *suri)
      * back on a sane default. */
     char *temp_default_packet_size;
     if ((ConfGet("default-packet-size", &temp_default_packet_size)) != 1) {
+        int lthread;
+        int nlive;
         switch (suri->run_mode) {
             case RUNMODE_PCAP_DEV:
             case RUNMODE_AFP_DEV:
             case RUNMODE_NETMAP:
             case RUNMODE_PFRING:
-                /* FIXME this don't work effficiently in multiinterface */
-                /* find payload for interface and use it */
-                default_packet_size = GetIfaceMaxPacketSize(suri->pcap_dev);
+                nlive = LiveGetDeviceCount();
+                for (lthread = 0; lthread < nlive; lthread++) {
+                    char *live_dev = LiveGetDeviceName(lthread);
+                    unsigned int iface_max_packet_size = GetIfaceMaxPacketSize(live_dev);
+                    if (iface_max_packet_size > default_packet_size)
+                        default_packet_size = iface_max_packet_size;
+                }
                 if (default_packet_size)
                     break;
                 /* fall through */
@@ -2226,6 +2232,11 @@ int main(int argc, char **argv)
         exit(EXIT_FAILURE);
     }
 
+    if (suri.run_mode == RUNMODE_DUMP_CONFIG) {
+        ConfDump();
+        exit(EXIT_SUCCESS);
+    }
+
     /* Since our config is now loaded we can finish configurating the
      * logging module. */
     SCLogLoadConfig(suri.daemon, suri.verbose);
@@ -2234,9 +2245,8 @@ int main(int argc, char **argv)
 
     UtilCpuPrintSummary();
 
-    if (suri.run_mode == RUNMODE_DUMP_CONFIG) {
-        ConfDump();
-        exit(EXIT_SUCCESS);
+    if (ParseInterfacesList(suri.run_mode, suri.pcap_dev) != TM_ECODE_OK) {
+        exit(EXIT_FAILURE);
     }
 
     if (PostConfLoadedSetup(&suri) != TM_ECODE_OK) {
@@ -2320,10 +2330,6 @@ int main(int argc, char **argv)
         StatsSetupPostConfig();
     }
 
-    if (ParseInterfacesList(suri.run_mode, suri.pcap_dev) != TM_ECODE_OK) {
-        exit(EXIT_FAILURE);
-    }
-
     if(suri.run_mode == RUNMODE_CONF_TEST){
         SCLogNotice("Configuration provided was successfully loaded. Exiting.");
         exit(EXIT_SUCCESS);
@@ -2367,6 +2373,7 @@ int main(int argc, char **argv)
     }
 
     (void) SC_ATOMIC_CAS(&engine_stage, SURICATA_INIT, SURICATA_RUNTIME);
+    PacketPoolPostRunmodes();
 
     /* Un-pause all the paused threads */
     TmThreadContinueThreads();
@@ -2379,7 +2386,7 @@ int main(int argc, char **argv)
 
     if (suri.delayed_detect) {
         /* force 'reload', this will load the rules and swap engines */
-        DetectEngineReload(NULL);
+        DetectEngineReload(NULL, &suri);
 
         if (suri.sig_file != NULL)
             UtilSignalHandlerSetup(SIGUSR2, SignalHandlerSigusr2SigFileStartup);
@@ -2412,10 +2419,10 @@ int main(int argc, char **argv)
         }
 
         if (sigusr2_count > 0) {
-            DetectEngineReload(conf_filename);
+            DetectEngineReload(conf_filename, &suri);
             sigusr2_count--;
         } else if (DetectEngineReloadIsStart()) {
-            DetectEngineReload(conf_filename);
+            DetectEngineReload(conf_filename, &suri);
             DetectEngineReloadSetDone();
         }
 
diff --git a/framework/src/suricata/src/threadvars.h b/framework/src/suricata/src/threadvars.h
index a632a23b..79a2b34d 100644
--- a/framework/src/suricata/src/threadvars.h
+++ b/framework/src/suricata/src/threadvars.h
@@ -71,6 +71,9 @@ typedef struct ThreadVars_ {
     /** no of times the thread has been restarted on failure */
     uint8_t restarted;
 
+    /** TmModule::flags for each module part of this thread */
+    uint8_t tmm_flags;
+
     /** local id */
     int id;
 
diff --git a/framework/src/suricata/src/tm-threads.c b/framework/src/suricata/src/tm-threads.c
index c6828b40..2b26404f 100644
--- a/framework/src/suricata/src/tm-threads.c
+++ b/framework/src/suricata/src/tm-threads.c
@@ -730,6 +730,7 @@ static inline TmSlot * _TmSlotSetFuncAppend(ThreadVars *tv, TmModule *tm, void *
      * received a TM as arg, if it didn't exist */
     slot->tm_id = TmModuleGetIDForTM(tm);
 
+    tv->tmm_flags |= tm->flags;
     tv->cap_flags |= tm->cap_flags;
 
     if (tv->tm_slots == NULL) {
@@ -2033,6 +2034,29 @@ ThreadVars *TmThreadsGetCallingThread(void)
     return NULL;
 }
 
+/**
+ * \brief returns a count of all the threads that match the flag
+ */
+uint32_t TmThreadCountThreadsByTmmFlags(uint8_t flags)
+{
+    ThreadVars *tv = NULL;
+    int i = 0;
+    uint32_t cnt = 0;
+
+    SCMutexLock(&tv_root_lock);
+    for (i = 0; i < TVT_MAX; i++) {
+        tv = tv_root[i];
+        while (tv != NULL) {
+            if ((tv->tmm_flags & flags) == flags)
+                cnt++;
+
+            tv = tv->next;
+        }
+    }
+    SCMutexUnlock(&tv_root_lock);
+    return cnt;
+}
+
 typedef struct Thread_ {
     ThreadVars *tv;     /**< threadvars structure */
     const char *name;
diff --git a/framework/src/suricata/src/tm-threads.h b/framework/src/suricata/src/tm-threads.h
index 9b9fd620..397fbd2e 100644
--- a/framework/src/suricata/src/tm-threads.h
+++ b/framework/src/suricata/src/tm-threads.h
@@ -132,6 +132,8 @@ void TmThreadDisablePacketThreads(void);
 void TmThreadDisableReceiveThreads(void);
 TmSlot *TmThreadGetFirstTmSlotForPartialPattern(const char *);
 
+uint32_t TmThreadCountThreadsByTmmFlags(uint8_t flags);
+
 /**
  *  \brief Process the rest of the functions (if any) and queue.
  */
diff --git a/framework/src/suricata/src/tmqh-packetpool.c b/framework/src/suricata/src/tmqh-packetpool.c
index a1f19dca..75139254 100644
--- a/framework/src/suricata/src/tmqh-packetpool.c
+++ b/framework/src/suricata/src/tmqh-packetpool.c
@@ -38,6 +38,8 @@
 #include "stream-tcp-reassemble.h"
 
 #include "tm-queuehandlers.h"
+#include "tm-threads.h"
+#include "tm-modules.h"
 
 #include "pkt-var.h"
 
@@ -50,6 +52,7 @@
 
 /* Number of freed packet to save for one pool before freeing them. */
 #define MAX_PENDING_RETURN_PACKETS 32
+static uint32_t max_pending_return_packets = MAX_PENDING_RETURN_PACKETS;
 
 #ifdef TLS
 __thread PktPool thread_pkt_pool;
@@ -313,7 +316,7 @@ void PacketPoolReturnPacket(Packet *p)
             p->next = my_pool->pending_head;
             my_pool->pending_head = p;
             my_pool->pending_count++;
-            if (SC_ATOMIC_GET(pool->return_stack.sync_now) || my_pool->pending_count > MAX_PENDING_RETURN_PACKETS) {
+            if (SC_ATOMIC_GET(pool->return_stack.sync_now) || my_pool->pending_count > max_pending_return_packets) {
                 /* Return the entire list of pending packets. */
                 SCMutexLock(&pool->return_stack.mutex);
                 my_pool->pending_tail->next = pool->return_stack.head;
@@ -563,3 +566,34 @@ void TmqhReleasePacketsToPacketPool(PacketQueue *pq)
 
     return;
 }
+
+/**
+ *  \brief Set the max_pending_return_packets value
+ *
+ *  Set it to the max pending packets value, devided by the number
+ *  of lister threads. Normally, in autofp these are the stream/detect/log
+ *  worker threads.
+ *
+ *  The max_pending_return_packets value needs to stay below the packet
+ *  pool size of the 'producers' (normally pkt capture threads but also
+ *  flow timeout injection ) to avoid a deadlock where all the 'workers'
+ *  keep packets in their return pools, while the capture thread can't
+ *  continue because its pool is empty.
+ */
+void PacketPoolPostRunmodes(void)
+{
+    extern intmax_t max_pending_packets;
+
+    uint32_t threads = TmThreadCountThreadsByTmmFlags(TM_FLAG_DETECT_TM);
+    if (threads == 0)
+        return;
+    if (threads > max_pending_packets)
+        return;
+
+    uint32_t packets = (max_pending_packets / threads) - 1;
+    if (packets < max_pending_return_packets)
+        max_pending_return_packets = packets;
+
+    SCLogDebug("detect threads %u, max packets %u, max_pending_return_packets %u",
+            threads, (uint)threads, max_pending_return_packets);
+}
diff --git a/framework/src/suricata/src/tmqh-packetpool.h b/framework/src/suricata/src/tmqh-packetpool.h
index ab45184c..2b6b90b0 100644
--- a/framework/src/suricata/src/tmqh-packetpool.h
+++ b/framework/src/suricata/src/tmqh-packetpool.h
@@ -78,5 +78,6 @@ void PacketPoolReturnPacket(Packet *p);
 void PacketPoolInit(void);
 void PacketPoolInitEmpty(void);
 void PacketPoolDestroy(void);
+void PacketPoolPostRunmodes(void);
 
 #endif /* __TMQH_PACKETPOOL_H__ */
diff --git a/framework/src/suricata/src/unix-manager.c b/framework/src/suricata/src/unix-manager.c
index 1960df56..9357f4c5 100644
--- a/framework/src/suricata/src/unix-manager.c
+++ b/framework/src/suricata/src/unix-manager.c
@@ -356,7 +356,7 @@ int UnixCommandAccept(UnixCommand *this)
     json_decref(server_msg);
 
     /* client connected */
-    SCLogInfo("Unix socket: client connected");
+    SCLogDebug("Unix socket: client connected");
 
     uclient = SCMalloc(sizeof(UnixClient));
     if (unlikely(uclient == NULL)) {
@@ -478,9 +478,9 @@ void UnixCommandRun(UnixCommand * this, UnixClient *client)
     ret = recv(client->fd, buffer, sizeof(buffer) - 1, 0);
     if (ret <= 0) {
         if (ret == 0) {
-            SCLogInfo("Unix socket: lost connection with client");
+            SCLogDebug("Unix socket: lost connection with client");
         } else {
-            SCLogInfo("Unix socket: error on recv() from client: %s",
+            SCLogError(SC_ERR_SOCKET, "Unix socket: error on recv() from client: %s",
                       strerror(errno));
         }
         UnixCommandClose(this, client->fd);
@@ -525,7 +525,7 @@ int UnixMain(UnixCommand * this)
         if (errno == EINTR) {
             return 1;
         }
-        SCLogInfo("Command server: select() fatal error: %s", strerror(errno));
+        SCLogError(SC_ERR_SOCKET, "Command server: select() fatal error: %s", strerror(errno));
         return 0;
     }
 
diff --git a/framework/src/suricata/src/util-debug.c b/framework/src/suricata/src/util-debug.c
index f1ac0463..d9af08c4 100644
--- a/framework/src/suricata/src/util-debug.c
+++ b/framework/src/suricata/src/util-debug.c
@@ -776,7 +776,7 @@ static inline SCLogOPIfaceCtx *SCLogInitConsoleOPIface(const char *log_format,
     }
     iface_ctx->log_level = tmp_log_level;
 
-    if (isatty(fileno(stdout))) {
+    if (isatty(fileno(stdout)) && isatty(fileno(stderr))) {
         iface_ctx->use_color = TRUE;
     }
 
diff --git a/framework/src/suricata/src/util-decode-mime.c b/framework/src/suricata/src/util-decode-mime.c
index 3f4affcc..51d1468e 100644
--- a/framework/src/suricata/src/util-decode-mime.c
+++ b/framework/src/suricata/src/util-decode-mime.c
@@ -79,9 +79,8 @@
 #define MAX_IP6_CHARS  39
 
 /* Globally hold configuration data */
-static MimeDecConfig mime_dec_config = { 1, 1, 1, MAX_HEADER_VALUE };
+static MimeDecConfig mime_dec_config = { 1, 1, 1, 0, MAX_HEADER_VALUE };
 
-#ifdef DEBUG
 /* Mime Parser String translation */
 static const char *StateFlags[] = { "NONE",
         "HEADER_READY",
@@ -93,7 +92,6 @@ static const char *StateFlags[] = { "NONE",
         "PARSE_DONE",
         "PARSE_ERROR",
         NULL };
-#endif
 
 /* URL executable file extensions */
 static const char *UrlExeExts[] = { ".exe",
@@ -299,6 +297,35 @@ MimeDecField * MimeDecAddField(MimeDecEntity *entity)
     return node;
 }
 
+
+/**
+ * \brief Searches for header fields with the specified name
+ *
+ * \param entity The entity to search
+ * \param name The header name (lowercase)
+ *
+ * \return number of items found
+ *
+ */
+int MimeDecFindFieldsForEach(const MimeDecEntity *entity, const char *name, int (*DataCallback)(const uint8_t *val, const size_t, void *data), void *data)
+{
+    MimeDecField *curr = entity->field_list;
+    int found = 0;
+
+    while (curr != NULL) {
+        /* name is stored lowercase */
+        if (strlen(name) == curr->name_len) {
+            if (SCMemcmp(curr->name, name, curr->name_len) == 0) {
+                if (DataCallback(curr->value, curr->value_len, data))
+                    found++;
+            }
+        }
+        curr = curr->next;
+    }
+
+    return found;
+}
+
 /**
  * \brief Searches for a header field with the specified name
  *
@@ -915,7 +942,7 @@ static int IsIpv4Host(const uint8_t *urlhost, uint32_t len)
  */
 static int IsIpv6Host(const uint8_t *urlhost, uint32_t len)
 {
-    struct sockaddr_in sa;
+    struct in6_addr in6;
     char tempIp[MAX_IP6_CHARS + 1];
 
     /* Cut off at '/'  */
@@ -936,7 +963,7 @@ static int IsIpv6Host(const uint8_t *urlhost, uint32_t len)
     memcpy(tempIp, urlhost, i);
     tempIp[i] = '\0';
 
-    return inet_pton(AF_INET6, tempIp, &(sa.sin_addr));
+    return inet_pton(AF_INET6, tempIp, &in6);
 }
 
 /**
@@ -1992,6 +2019,7 @@ static int ProcessMimeHeaders(const uint8_t *buf, uint32_t len,
  *
  * \return MIME_DEC_OK on success, otherwise < 0 on failure
  */
+
 static int ProcessBodyComplete(MimeDecParseState *state)
 {
     int ret = MIME_DEC_OK;
@@ -2012,6 +2040,13 @@ static int ProcessBodyComplete(MimeDecParseState *state)
         }
     }
 
+#ifdef HAVE_NSS
+    if (state->md5_ctx) {
+        unsigned int len = 0;
+        HASH_End(state->md5_ctx, state->md5, &len, sizeof(state->md5));
+    }
+#endif
+
     /* Invoke pre-processor and callback with remaining data */
     ret = ProcessDecodedDataChunk(state->data_chunk, state->data_chunk_len, state);
     if (ret != MIME_DEC_OK) {
@@ -2180,6 +2215,18 @@ static int ProcessMimeBody(const uint8_t *buf, uint32_t len,
     int body_found = 0;
     uint32_t tlen;
 
+#ifdef HAVE_NSS
+    if (MimeDecGetConfig()->body_md5) {
+        if (state->body_begin == 1) {
+            if (state->md5_ctx == NULL) {
+                state->md5_ctx = HASH_Create(HASH_AlgMD5);
+                HASH_Begin(state->md5_ctx);
+            }
+        }
+        HASH_Update(state->md5_ctx, buf, len + state->current_line_delimiter_len);
+    }
+#endif
+
     /* Ignore empty lines */
     if (len == 0) {
         return ret;
@@ -2254,6 +2301,11 @@ static int ProcessMimeBody(const uint8_t *buf, uint32_t len,
     return ret;
 }
 
+const char *MimeDecParseStateGetStatus(MimeDecParseState *state)
+{
+    return StateFlags[state->state_flag];
+}
+
 /**
  * \brief Processes the MIME Entity based on the input line and current state of
  * the parser
@@ -2394,6 +2446,10 @@ void MimeDecDeInitParser(MimeDecParseState *state)
     SCFree(state->hname);
     FreeDataValue(state->hvalue);
     FreeMimeDecStack(state->stack);
+#ifdef HAVE_NSS
+    if (state->md5_ctx)
+        HASH_Destroy(state->md5_ctx);
+#endif
     SCFree(state);
 }
 
@@ -2459,12 +2515,13 @@ int MimeDecParseComplete(MimeDecParseState *state)
  *
  * \param line A string representing the line (w/out CRLF)
  * \param len The length of the line
+ * \param delim_len The length of the line end delimiter
  * \param state The parser state
  *
  * \return MIME_DEC_OK on success, otherwise < 0 on failure
  */
 int MimeDecParseLine(const uint8_t *line, const uint32_t len,
-        MimeDecParseState *state)
+        const uint8_t delim_len, MimeDecParseState *state)
 {
     int ret = MIME_DEC_OK;
 
@@ -2475,6 +2532,7 @@ int MimeDecParseLine(const uint8_t *line, const uint32_t len,
         SCLogDebug("SMTP LINE - EMPTY");
     }
 
+    state->current_line_delimiter_len = delim_len;
     /* Process the entity */
     ret = ProcessMimeEntity(line, len, state);
     if (ret != MIME_DEC_OK) {
@@ -2522,8 +2580,10 @@ MimeDecEntity * MimeDecParseFullMsg(const uint8_t *buf, uint32_t blen, void *dat
 
             line = tok;
 
+            state->current_line_delimiter_len = (remainPtr - tok) - tokLen;
             /* Parse the line */
-            ret = MimeDecParseLine(line, tokLen, state);
+            ret = MimeDecParseLine(line, tokLen,
+                                   (remainPtr - tok) - tokLen, state);
             if (ret != MIME_DEC_OK) {
                 SCLogDebug("Error: MimeDecParseLine() function failed: %d",
                         ret);
@@ -2610,25 +2670,25 @@ static int MimeDecParseLineTest01(void)
             TestDataChunkCallback);
 
     char *str = "From: Sender1";
-    ret |= MimeDecParseLine((uint8_t *)str, strlen(str), state);
+    ret |= MimeDecParseLine((uint8_t *)str, strlen(str), 1, state);
 
     str = "To: Recipient1";
-    ret |= MimeDecParseLine((uint8_t *)str, strlen(str), state);
+    ret |= MimeDecParseLine((uint8_t *)str, strlen(str), 1, state);
 
     str = "Content-Type: text/plain";
-    ret |= MimeDecParseLine((uint8_t *)str, strlen(str), state);
+    ret |= MimeDecParseLine((uint8_t *)str, strlen(str), 1, state);
 
     str = "";
-    ret |= MimeDecParseLine((uint8_t *)str, strlen(str), state);
+    ret |= MimeDecParseLine((uint8_t *)str, strlen(str), 1, state);
 
     str = "A simple message line 1";
-    ret |= MimeDecParseLine((uint8_t *)str, strlen(str), state);
+    ret |= MimeDecParseLine((uint8_t *)str, strlen(str), 1, state);
 
     str = "A simple message line 2";
-    ret |= MimeDecParseLine((uint8_t *)str, strlen(str), state);
+    ret |= MimeDecParseLine((uint8_t *)str, strlen(str), 1, state);
 
     str = "A simple message line 3";
-    ret |= MimeDecParseLine((uint8_t *)str, strlen(str), state);
+    ret |= MimeDecParseLine((uint8_t *)str, strlen(str), 1, state);
 
     if (ret != MIME_DEC_OK) {
         return ret;
@@ -2678,23 +2738,23 @@ static int MimeDecParseLineTest02(void)
             TestDataChunkCallback);
 
     char *str = "From: Sender1";
-    ret |= MimeDecParseLine((uint8_t *)str, strlen(str), state);
+    ret |= MimeDecParseLine((uint8_t *)str, strlen(str), 1, state);
 
     str = "To: Recipient1";
-    ret |= MimeDecParseLine((uint8_t *)str, strlen(str), state);
+    ret |= MimeDecParseLine((uint8_t *)str, strlen(str), 1, state);
 
     str = "Content-Type: text/plain";
-    ret |= MimeDecParseLine((uint8_t *)str, strlen(str), state);
+    ret |= MimeDecParseLine((uint8_t *)str, strlen(str), 1, state);
 
     str = "";
-    ret |= MimeDecParseLine((uint8_t *)str, strlen(str), state);
+    ret |= MimeDecParseLine((uint8_t *)str, strlen(str), 1, state);
 
     str = "A simple message line 1";
-    ret |= MimeDecParseLine((uint8_t *)str, strlen(str), state);
+    ret |= MimeDecParseLine((uint8_t *)str, strlen(str), 1, state);
 
     str = "A simple message line 2 click on http://www.test.com/malware.exe?"
             "hahah hopefully you click this link";
-    ret |= MimeDecParseLine((uint8_t *)str, strlen(str), state);
+    ret |= MimeDecParseLine((uint8_t *)str, strlen(str), 1, state);
 
     if (ret != MIME_DEC_OK) {
         return ret;
@@ -2762,6 +2822,59 @@ static int MimeDecParseFullMsgTest01(void)
     return ret;
 }
 
+/* Test full message with linebreaks */
+static int MimeDecParseFullMsgTest02(void)
+{
+    int ret = MIME_DEC_OK;
+
+    uint32_t expected_count = 3;
+    uint32_t line_count = 0;
+
+    char msg[] = "From: Sender2\r\n"
+            "To: Recipient2\r\n"
+            "Subject: subject2\r\n"
+            "Content-Type: text/plain\r\n"
+            "\r\n"
+            "Line 1\r\n"
+            "Line 2\r\n"
+            "Line 3\r\n";
+
+    MimeDecEntity *entity = MimeDecParseFullMsg((uint8_t *)msg, strlen(msg), &line_count,
+            TestDataChunkCallback);
+
+    if (entity == NULL) {
+        SCLogInfo("Warning: Message failed to parse");
+        return -1;
+    }
+
+    MimeDecField *field = MimeDecFindField(entity, "subject");
+    if (field == NULL) {
+        SCLogInfo("Warning: Message failed to parse");
+        return -1;
+    }
+
+    if (field->value_len != sizeof("subject2") - 1) {
+        SCLogInfo("Warning: failed to get subject");
+        return -1;
+    }
+
+    if (memcmp(field->value, "subject2", field->value_len) != 0) {
+        SCLogInfo("Warning: failed to get subject");
+        return -1;
+    }
+
+
+    MimeDecFreeEntity(entity);
+
+    if (expected_count != line_count) {
+        SCLogInfo("Warning: Line count is invalid: expected - %d actual - %d",
+                expected_count, line_count);
+        return -1;
+    }
+
+    return ret;
+}
+
 static int MimeBase64DecodeTest01(void)
 {
     int ret = -1;
@@ -2771,7 +2884,7 @@ static int MimeBase64DecodeTest01(void)
     char *base64msg = "YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUZHSElKS0xNTk9QU"
             "VJTVFVWV1hZWjEyMzQ1Njc4OTBAIyQlXiYqKCktPV8rLC4vOydbXTw+Pzo=";
 
-    uint8_t *dst = SCMalloc(strlen(msg)-1);
+    uint8_t *dst = SCMalloc(strlen(msg) + 1);
     if (dst == NULL)
         return 0;
 
@@ -2781,6 +2894,8 @@ static int MimeBase64DecodeTest01(void)
         ret = 0;
     }
 
+    SCFree(dst);
+
     return ret;
 }
 
@@ -2879,6 +2994,7 @@ void MimeDecRegisterTests(void)
     UtRegisterTest("MimeDecParseLineTest01", MimeDecParseLineTest01, 0);
     UtRegisterTest("MimeDecParseLineTest02", MimeDecParseLineTest02, 0);
     UtRegisterTest("MimeDecParseFullMsgTest01", MimeDecParseFullMsgTest01, 0);
+    UtRegisterTest("MimeDecParseFullMsgTest02", MimeDecParseFullMsgTest02, 0);
     UtRegisterTest("MimeBase64DecodeTest01", MimeBase64DecodeTest01, 0);
     UtRegisterTest("MimeIsExeURLTest01", MimeIsExeURLTest01, 0);
     UtRegisterTest("MimeIsIpv4HostTest01", MimeIsIpv4HostTest01, 0);
diff --git a/framework/src/suricata/src/util-decode-mime.h b/framework/src/suricata/src/util-decode-mime.h
index 792fbcc1..02b3bb13 100644
--- a/framework/src/suricata/src/util-decode-mime.h
+++ b/framework/src/suricata/src/util-decode-mime.h
@@ -33,9 +33,6 @@
 #include "util-base64.h"
 #include "util-debug.h"
 
-/* Header Flags */
-#define HDR_IS_LOGGED         1
-
 /* Content Flags */
 #define CTNT_IS_MSG           1
 #define CTNT_IS_ENV           2
@@ -64,7 +61,7 @@
 #define ANOM_MALFORMED_MSG      64  /* Misc msg format errors found */
 #define ANOM_LONG_BOUNDARY     128  /* Boundary too long */
 
-/* Pubicly exposed size constants */
+/* Publicly exposed size constants */
 #define DATA_CHUNK_SIZE  3072  /* Should be divisible by 3 */
 #define LINEREM_SIZE      256
 
@@ -97,6 +94,7 @@ typedef struct MimeDecConfig {
     int decode_base64;  /**< Decode base64 bodies */
     int decode_quoted_printable;  /**< Decode quoted-printable bodies */
     int extract_urls;  /**< Extract and store URLs in data structure */
+    int body_md5;  /**< Compute md5 sum of body */
     uint32_t header_value_depth;  /**< Depth of which to store header values
                                        (Default is 2000) */
 } MimeDecConfig;
@@ -197,11 +195,16 @@ typedef struct MimeDecParseState {
     uint8_t bvremain[B64_BLOCK];  /**< Remainder from base64-decoded line */
     uint8_t bvr_len;  /**< Length of remainder from base64-decoded line */
     uint8_t data_chunk[DATA_CHUNK_SIZE];  /**< Buffer holding data chunk */
+#ifdef HAVE_NSS
+    HASHContext *md5_ctx;
+    uint8_t md5[MD5_LENGTH];
+#endif
     uint8_t state_flag;  /**<  Flag representing current state of parser */
     uint32_t data_chunk_len;  /**< Length of data chunk */
     int found_child;  /**< Flag indicating a child entity was found */
     int body_begin;  /**< Currently at beginning of body */
     int body_end;  /**< Currently at end of body */
+    uint8_t current_line_delimiter_len; /**< Length of line delimiter */
     void *data;  /**< Pointer to data specific to the caller */
     int (*DataChunkProcessorFunc) (const uint8_t *chunk, uint32_t len,
             struct MimeDecParseState *state);  /**< Data chunk processing function callback */
@@ -219,6 +222,7 @@ void MimeDecFreeUrl(MimeDecUrl *url);
 /* List functions */
 MimeDecField * MimeDecAddField(MimeDecEntity *entity);
 MimeDecField * MimeDecFindField(const MimeDecEntity *entity, const char *name);
+int MimeDecFindFieldsForEach(const MimeDecEntity *entity, const char *name, int (*DataCallback)(const uint8_t *val, const size_t, void *data), void *data);
 MimeDecEntity * MimeDecAddEntity(MimeDecEntity *parent);
 
 /* Helper functions */
@@ -230,9 +234,10 @@ MimeDecParseState * MimeDecInitParser(void *data, int (*dcpfunc)(const uint8_t *
         uint32_t len, MimeDecParseState *state));
 void MimeDecDeInitParser(MimeDecParseState *state);
 int MimeDecParseComplete(MimeDecParseState *state);
-int MimeDecParseLine(const uint8_t *line, const uint32_t len, MimeDecParseState *state);
+int MimeDecParseLine(const uint8_t *line, const uint32_t len, const uint8_t delim_len, MimeDecParseState *state);
 MimeDecEntity * MimeDecParseFullMsg(const uint8_t *buf, uint32_t blen, void *data,
         int (*DataChunkProcessorFunc)(const uint8_t *chunk, uint32_t len, MimeDecParseState *state));
+const char *MimeDecParseStateGetStatus(MimeDecParseState *state);
 
 /* Test functions */
 void MimeDecRegisterTests(void);
diff --git a/framework/src/suricata/src/util-error.c b/framework/src/suricata/src/util-error.c
index 6e783f42..461e1870 100644
--- a/framework/src/suricata/src/util-error.c
+++ b/framework/src/suricata/src/util-error.c
@@ -309,6 +309,7 @@ const char * SCErrorToString(SCError err)
         CASE_CODE (SC_ERR_IPPAIR_INIT);
         CASE_CODE (SC_ERR_MT_NO_SELECTOR);
         CASE_CODE (SC_ERR_MT_DUPLICATE_TENANT);
+        CASE_CODE (SC_ERR_NO_JSON_SUPPORT);
     }
 
     return "UNKNOWN_ERROR";
diff --git a/framework/src/suricata/src/util-error.h b/framework/src/suricata/src/util-error.h
index d1e42a8c..b7df4766 100644
--- a/framework/src/suricata/src/util-error.h
+++ b/framework/src/suricata/src/util-error.h
@@ -298,6 +298,7 @@ typedef enum {
     SC_ERR_IPPAIR_INIT,
     SC_ERR_MT_NO_SELECTOR,
     SC_ERR_MT_DUPLICATE_TENANT,
+    SC_ERR_NO_JSON_SUPPORT,
 } SCError;
 
 const char *SCErrorToString(SCError);
diff --git a/framework/src/suricata/src/util-host-os-info.c b/framework/src/suricata/src/util-host-os-info.c
index ad0de7e2..f04d6326 100644
--- a/framework/src/suricata/src/util-host-os-info.c
+++ b/framework/src/suricata/src/util-host-os-info.c
@@ -450,12 +450,12 @@ int SCHInfoTestInvalidIPV4Address02(void)
 {
     SCHInfoCreateContextBackup();
 
-    int result = 1;
+    int result = 0;
 
     if (SCHInfoAddHostOSInfo("linux", "192.168.1.566", SC_HINFO_IS_IPV4) != -1) {
         goto end;
     }
-    if (SCHInfoAddHostOSInfo("linux", "192.168.1", SC_HINFO_IS_IPV4 != -1)) {
+    if (SCHInfoAddHostOSInfo("linux", "192.168.1", SC_HINFO_IS_IPV4) != -1) {
         goto end;
     }
     if (SCHInfoAddHostOSInfo("linux", "192.", SC_HINFO_IS_IPV4) != -1) {
@@ -488,7 +488,7 @@ int SCHInfoTestInvalidIPV6Address03(void)
 {
     SCHInfoCreateContextBackup();
 
-    int result = 1;
+    int result = 0;
 
     if (SCHInfoAddHostOSInfo("linux", "2362:7322", SC_HINFO_IS_IPV6) != -1) {
         goto end;
@@ -530,7 +530,7 @@ int SCHInfoTestValidIPV4Address04(void)
 {
     SCHInfoCreateContextBackup();
 
-    int result = 1;
+    int result = 0;
 
     if (SCHInfoAddHostOSInfo("linux", "192.168.1.1", SC_HINFO_IS_IPV4) == -1) {
         goto end;
@@ -643,7 +643,7 @@ int SCHInfoTestValidIPV4Address05(void)
     SCHInfoCreateContextBackup();
 
     struct in_addr in;
-    int result = 1;
+    int result = 0;
 
     if (SCHInfoAddHostOSInfo("linux", "192.168.1.1", SC_HINFO_IS_IPV4) == -1) {
         goto end;
@@ -783,7 +783,7 @@ int SCHInfoTestValidIPV6Address06(void)
 {
     SCHInfoCreateContextBackup();
 
-    int result = 1;
+    int result = 0;
 
     if (SCHInfoAddHostOSInfo("linux",
                              "2351:2512:6211:6246:235A:6242:2352:62AD",
@@ -919,7 +919,7 @@ int SCHInfoTestValidIPV6Address07(void)
 {
     SCHInfoCreateContextBackup();
 
-    int result = 1;
+    int result = 0;
 
     if (SCHInfoAddHostOSInfo("linux",
                              "2351:2512:6211:6246:235A:6242:2352:62AD",
@@ -1076,7 +1076,7 @@ int SCHInfoTestValidIPV6Address08(void)
     SCHInfoCreateContextBackup();
 
     struct in6_addr in6;
-    int result = 1;
+    int result = 0;
 
     if (SCHInfoAddHostOSInfo("linux",
                              "2351:2512:6211:6246:235A:6242:2352:62AD",
@@ -1248,10 +1248,10 @@ int SCHInfoTestValidIPV4Address09(void)
 {
     SCHInfoCreateContextBackup();
 
-    int result = 1;
+    int result = 0;
 
     if (SCHInfoAddHostOSInfo("linux", "192.168.1.0", SC_HINFO_IS_IPV4) == -1) {
-                goto end;
+        goto end;
     }
     if (SCHInfoAddHostOSInfo("windows", "192.192.1.2", SC_HINFO_IS_IPV4) == -1) {
         goto end;
@@ -1319,22 +1319,26 @@ int SCHInfoTestValidIPV4Address09(void)
         goto end;
     }
 
-    if (SCHInfoGetHostOSFlavour("192.168.1.100") ==
+    /* 192.168.1.100 should match "macos" as its more specific than
+     * "solaris". */
+    if (SCHInfoGetHostOSFlavour("192.168.1.100") !=
         SCMapEnumNameToValue("macos", sc_hinfo_os_policy_map)) {
         goto end;
     }
 
+    /* Remove the 192.168.1.0/20 -> macos entry. */
     bzero(&servaddr, sizeof(servaddr));
     if (inet_pton(AF_INET, "192.168.0.0", &servaddr.sin_addr) <= 0) {
         goto end;
     }
     SCRadixRemoveKeyIPV4Netblock((uint8_t *)&servaddr.sin_addr, sc_hinfo_tree, 20);
 
-    if (SCHInfoGetHostOSFlavour("192.168.1.100") ==
+    if (SCHInfoGetHostOSFlavour("192.168.1.100") !=
         SCMapEnumNameToValue("solaris", sc_hinfo_os_policy_map)) {
         goto end;
     }
 
+    /* Remove the 192.168.1.0/16 -> solaris entry. */
     bzero(&servaddr, sizeof(servaddr));
     if (inet_pton(AF_INET, "192.168.0.0", &servaddr.sin_addr) <= 0) {
         goto end;
@@ -1481,7 +1485,7 @@ host-os-policy:\n\
   vista: [0.0.0.5]\n\
 \n";
 
-    int result = 1;
+    int result = 0;
 
     SCHInfoCreateContextBackup();
 
@@ -1527,7 +1531,7 @@ host-os-policy:\n\
   vista: [0.0.0.5]\n\
 \n";
 
-    int result = 1;
+    int result = 0;
 
     SCHInfoCreateContextBackup();
 
diff --git a/framework/src/suricata/src/util-logopenfile.c b/framework/src/suricata/src/util-logopenfile.c
index b25c4a82..65b80fac 100644
--- a/framework/src/suricata/src/util-logopenfile.c
+++ b/framework/src/suricata/src/util-logopenfile.c
@@ -34,6 +34,9 @@
 #include "util-logopenfile.h"
 #include "util-logopenfile-tile.h"
 
+const char * redis_push_cmd = "LPUSH";
+const char * redis_publish_cmd = "PUBLISH";
+
 /** \brief connect to the indicated local stream socket, logging any errors
  *  \param path filesystem path to connect to
  *  \param log_err, non-zero if connect failure should be logged.
@@ -330,6 +333,138 @@ int SCConfLogReopen(LogFileCtx *log_ctx)
     return 0;
 }
 
+
+#ifdef HAVE_LIBHIREDIS
+
+static void SCLogFileCloseRedis(LogFileCtx *log_ctx)
+{
+    if (log_ctx->redis) {
+        redisReply *reply;
+        int i;
+        for (i = 0; i < log_ctx->redis_setup.batch_count; i++) {
+            redisGetReply(log_ctx->redis, (void **)&reply);
+            if (reply)
+                freeReplyObject(reply);
+        }
+        redisFree(log_ctx->redis);
+        log_ctx->redis = NULL;
+    }
+    log_ctx->redis_setup.tried = 0;
+    log_ctx->redis_setup.batch_count = 0;
+}
+
+int SCConfLogOpenRedis(ConfNode *redis_node, LogFileCtx *log_ctx)
+{
+    const char *redis_server = NULL;
+    const char *redis_port = NULL;
+    const char *redis_mode = NULL;
+    const char *redis_key = NULL;
+
+    if (redis_node) {
+        redis_server = ConfNodeLookupChildValue(redis_node, "server");
+        redis_port =  ConfNodeLookupChildValue(redis_node, "port");
+        redis_mode =  ConfNodeLookupChildValue(redis_node, "mode");
+        redis_key =  ConfNodeLookupChildValue(redis_node, "key");
+    }
+    if (!redis_server) {
+        redis_server = "127.0.0.1";
+        SCLogInfo("Using default redis server (127.0.0.1)");
+    }
+    if (!redis_port)
+        redis_port = "6379";
+    if (!redis_mode)
+        redis_mode = "list";
+    if (!redis_key)
+        redis_key = "suricata";
+    log_ctx->redis_setup.key = SCStrdup(redis_key);
+
+    if (!log_ctx->redis_setup.key) {
+        SCLogError(SC_ERR_MEM_ALLOC, "Unable to allocate redis key name");
+        exit(EXIT_FAILURE);
+    }
+
+    log_ctx->redis_setup.batch_size = 0;
+
+    ConfNode *pipelining = ConfNodeLookupChild(redis_node, "pipelining");
+    if (pipelining) {
+        int enabled = 0;
+        int ret;
+        intmax_t val;
+        ret = ConfGetChildValueBool(pipelining, "enabled", &enabled);
+        if (ret && enabled) {
+            ret = ConfGetChildValueInt(pipelining, "batch-size", &val);
+            if (ret) {
+                log_ctx->redis_setup.batch_size = val;
+            } else {
+                log_ctx->redis_setup.batch_size = 10;
+            }
+        }
+    }
+
+    if (!strcmp(redis_mode, "list")) {
+        log_ctx->redis_setup.command = redis_push_cmd;
+        if (!log_ctx->redis_setup.command) {
+            SCLogError(SC_ERR_MEM_ALLOC, "Unable to allocate redis key command");
+            exit(EXIT_FAILURE);
+        }
+    } else {
+        log_ctx->redis_setup.command = redis_publish_cmd;
+        if (!log_ctx->redis_setup.command) {
+            SCLogError(SC_ERR_MEM_ALLOC, "Unable to allocate redis key command");
+            exit(EXIT_FAILURE);
+        }
+    }
+    redisContext *c = redisConnect(redis_server, atoi(redis_port));
+    if (c != NULL && c->err) {
+        SCLogError(SC_ERR_SOCKET, "Error connecting to redis server: %s", c->errstr);
+        exit(EXIT_FAILURE);
+    }
+
+    /* store server params for reconnection */
+    log_ctx->redis_setup.server = SCStrdup(redis_server);
+    if (!log_ctx->redis_setup.server) {
+        SCLogError(SC_ERR_MEM_ALLOC, "Error allocating redis server string");
+        exit(EXIT_FAILURE);
+    }
+    log_ctx->redis_setup.port = atoi(redis_port);
+    log_ctx->redis_setup.tried = 0;
+
+    log_ctx->redis = c;
+
+    log_ctx->Close = SCLogFileCloseRedis;
+
+    return 0;
+}
+
+int SCConfLogReopenRedis(LogFileCtx *log_ctx)
+{
+    if (log_ctx->redis != NULL) {
+        redisFree(log_ctx->redis);
+        log_ctx->redis = NULL;
+    }
+
+    /* only try to reconnect once per second */
+    if (log_ctx->redis_setup.tried >= time(NULL)) {
+        return -1;
+    }
+
+    redisContext *c = redisConnect(log_ctx->redis_setup.server, log_ctx->redis_setup.port);
+    if (c != NULL && c->err) {
+        if (log_ctx->redis_setup.tried == 0) {
+            SCLogError(SC_ERR_SOCKET, "Error connecting to redis server: %s\n", c->errstr);
+        }
+        redisFree(c);
+        log_ctx->redis_setup.tried = time(NULL);
+        return -1;
+    }
+    log_ctx->redis = c;
+    log_ctx->redis_setup.tried = 0;
+    log_ctx->redis_setup.batch_count = 0;
+    return 0;
+}
+
+#endif
+
 /** \brief LogFileNewCtx() Get a new LogFileCtx
  *  \retval LogFileCtx * pointer if succesful, NULL if error
  *  */
@@ -348,6 +483,10 @@ LogFileCtx *LogFileNewCtx(void)
     lf_ctx->Write = SCLogFileWrite;
     lf_ctx->Close = SCLogFileClose;
 
+#ifdef HAVE_LIBHIREDIS
+    lf_ctx->redis_setup.batch_count = 0;
+#endif
+
     return lf_ctx;
 }
 
@@ -367,6 +506,17 @@ int LogFileFreeCtx(LogFileCtx *lf_ctx)
         SCMutexUnlock(&lf_ctx->fp_mutex);
     }
 
+#ifdef HAVE_LIBHIREDIS
+    if (lf_ctx->type == LOGFILE_TYPE_REDIS) {
+        if (lf_ctx->redis)
+            redisFree(lf_ctx->redis);
+        if (lf_ctx->redis_setup.server)
+            SCFree(lf_ctx->redis_setup.server);
+        if (lf_ctx->redis_setup.key)
+            SCFree(lf_ctx->redis_setup.key);
+    }
+#endif
+
     SCMutexDestroy(&lf_ctx->fp_mutex);
 
     if (lf_ctx->prefix != NULL)
@@ -375,9 +525,120 @@ int LogFileFreeCtx(LogFileCtx *lf_ctx)
     if(lf_ctx->filename != NULL)
         SCFree(lf_ctx->filename);
 
+    if (lf_ctx->sensor_name)
+        SCFree(lf_ctx->sensor_name);
+
     OutputUnregisterFileRotationFlag(&lf_ctx->rotation_flag);
 
     SCFree(lf_ctx);
 
     SCReturnInt(1);
 }
+
+#ifdef HAVE_LIBHIREDIS
+static int  LogFileWriteRedis(LogFileCtx *file_ctx, char *string, size_t string_len)
+{
+    if (file_ctx->redis == NULL) {
+        SCConfLogReopenRedis(file_ctx);
+        if (file_ctx->redis == NULL) {
+            return -1;
+        } else {
+            SCLogInfo("Reconnected to redis server");
+        }
+    }
+    /* TODO go async here ? */
+    if (file_ctx->redis_setup.batch_size) {
+        redisAppendCommand(file_ctx->redis, "%s %s %s",
+                file_ctx->redis_setup.command,
+                file_ctx->redis_setup.key,
+                string);
+        if (file_ctx->redis_setup.batch_count == file_ctx->redis_setup.batch_size) {
+            redisReply *reply;
+            int i;
+            file_ctx->redis_setup.batch_count = 0;
+            for (i = 0; i <= file_ctx->redis_setup.batch_size; i++) {
+                if (redisGetReply(file_ctx->redis, (void **)&reply) == REDIS_OK) {
+                    freeReplyObject(reply);
+                } else {
+                    if (file_ctx->redis->err) {
+                        SCLogInfo("Error when fetching reply: %s (%d)",
+                                file_ctx->redis->errstr,
+                                file_ctx->redis->err);
+                    }
+                    switch (file_ctx->redis->err) {
+                        case REDIS_ERR_EOF:
+                        case REDIS_ERR_IO:
+                            SCLogInfo("Reopening connection to redis server");
+                            SCConfLogReopenRedis(file_ctx);
+                            if (file_ctx->redis) {
+                                SCLogInfo("Reconnected to redis server");
+                                return 0;
+                            } else {
+                                SCLogInfo("Unable to reconnect to redis server");
+                                return 0;
+                            }
+                            break;
+                        default:
+                            SCLogWarning(SC_ERR_INVALID_VALUE,
+                                    "Unsupported error code %d",
+                                    file_ctx->redis->err);
+                            return 0;
+                    }
+                }
+            }
+        } else {
+            file_ctx->redis_setup.batch_count++;
+        }
+    } else {
+        redisReply *reply = redisCommand(file_ctx->redis, "%s %s %s",
+                file_ctx->redis_setup.command,
+                file_ctx->redis_setup.key,
+                string);
+
+        switch (reply->type) {
+            case REDIS_REPLY_ERROR:
+                SCLogWarning(SC_ERR_SOCKET, "Redis error: %s", reply->str);
+                SCConfLogReopenRedis(file_ctx);
+                break;
+            case REDIS_REPLY_INTEGER:
+                SCLogDebug("Redis integer %lld", reply->integer);
+                break;
+            default:
+                SCLogError(SC_ERR_INVALID_VALUE,
+                        "Redis default triggered with %d", reply->type);
+                SCConfLogReopenRedis(file_ctx);
+                break;
+        }
+        freeReplyObject(reply);
+    }
+    return 0;
+}
+#endif
+
+int LogFileWrite(LogFileCtx *file_ctx, MemBuffer *buffer)
+{
+    if (file_ctx->type == LOGFILE_TYPE_SYSLOG) {
+        syslog(file_ctx->syslog_setup.alert_syslog_level, "%s",
+                (const char *)MEMBUFFER_BUFFER(buffer));
+    } else if (file_ctx->type == LOGFILE_TYPE_FILE ||
+               file_ctx->type == LOGFILE_TYPE_UNIX_DGRAM ||
+               file_ctx->type == LOGFILE_TYPE_UNIX_STREAM)
+    {
+        /* append \n for files only */
+        MemBufferWriteString(buffer, "\n");
+        SCMutexLock(&file_ctx->fp_mutex);
+        file_ctx->Write((const char *)MEMBUFFER_BUFFER(buffer),
+                        MEMBUFFER_OFFSET(buffer), file_ctx);
+        SCMutexUnlock(&file_ctx->fp_mutex);
+    }
+#ifdef HAVE_LIBHIREDIS
+    else if (file_ctx->type == LOGFILE_TYPE_REDIS) {
+        SCMutexLock(&file_ctx->fp_mutex);
+        LogFileWriteRedis(file_ctx, (const char *)MEMBUFFER_BUFFER(buffer),
+                MEMBUFFER_OFFSET(buffer));
+        SCMutexUnlock(&file_ctx->fp_mutex);
+    }
+#endif
+
+    return 0;
+}
diff --git a/framework/src/suricata/src/util-logopenfile.h b/framework/src/suricata/src/util-logopenfile.h
index d345475d..f0a123ac 100644
--- a/framework/src/suricata/src/util-logopenfile.h
+++ b/framework/src/suricata/src/util-logopenfile.h
@@ -26,6 +26,11 @@
 
 #include "conf.h"            /* ConfNode   */
 #include "tm-modules.h"      /* LogFileCtx */
+#include "util-buffer.h"
+
+#ifdef HAVE_LIBHIREDIS
+#include "hiredis/hiredis.h"
+#endif
 
 typedef struct {
     uint16_t fileno;
@@ -34,13 +39,43 @@ typedef struct {
 enum LogFileType { LOGFILE_TYPE_FILE,
                    LOGFILE_TYPE_SYSLOG,
                    LOGFILE_TYPE_UNIX_DGRAM,
-                   LOGFILE_TYPE_UNIX_STREAM };
+                   LOGFILE_TYPE_UNIX_STREAM,
+                   LOGFILE_TYPE_REDIS };
+
+typedef struct SyslogSetup_ {
+    int alert_syslog_level;
+} SyslogSetup;
+
+#ifdef HAVE_LIBHIREDIS
+enum RedisMode { REDIS_LIST, REDIS_CHANNEL };
+
+typedef struct RedisSetup_ {
+    enum RedisMode mode;
+    const char *command;
+    char *key;
+    int  batch_size;
+    int  batch_count;
+    char *server;
+    int  port;
+    time_t tried;
+} RedisSetup;
+#endif
 
 /** Global structure for Output Context */
 typedef struct LogFileCtx_ {
     union {
         FILE *fp;
         PcieFile *pcie_fp;
+#ifdef HAVE_LIBHIREDIS
+        redisContext *redis;
+#endif
+    };
+
+    union {
+        SyslogSetup syslog_setup;
+#ifdef HAVE_LIBHIREDIS
+        RedisSetup redis_setup;
+#endif
     };
 
     int (*Write)(const char *buffer, int buffer_len, struct LogFileCtx_ *fp);
@@ -56,6 +91,9 @@ typedef struct LogFileCtx_ {
     /** The name of the file */
     char *filename;
 
+    /** Suricata sensor name */
+    char *sensor_name;
+
     /** Handle auto-connecting / reconnecting sockets */
     int is_sock;
     int sock_type;
@@ -92,8 +130,10 @@ typedef struct LogFileCtx_ {
 
 LogFileCtx *LogFileNewCtx(void);
 int LogFileFreeCtx(LogFileCtx *);
+int LogFileWrite(LogFileCtx *file_ctx, MemBuffer *buffer);
 
 int SCConfLogOpenGeneric(ConfNode *conf, LogFileCtx *, const char *, int);
+int SCConfLogOpenRedis(ConfNode *conf, LogFileCtx *log_ctx);
 int SCConfLogReopen(LogFileCtx *);
 
 #endif /* __UTIL_LOGOPENFILE_H__ */
diff --git a/framework/src/suricata/src/util-profiling-rules.c b/framework/src/suricata/src/util-profiling-rules.c
index 2f4ec5c7..945bc8b3 100644
--- a/framework/src/suricata/src/util-profiling-rules.c
+++ b/framework/src/suricata/src/util-profiling-rules.c
@@ -38,10 +38,6 @@
 
 #ifdef PROFILING
 
-#ifndef MIN
-#define MIN(a, b) (((a) < (b)) ? (a) : (b))
-#endif
-
 /**
  * Extra data for rule profiling.
  */
@@ -85,6 +81,7 @@ extern int profiling_output_to_file;
 int profiling_rules_enabled = 0;
 static char *profiling_file_name = "";
 static const char *profiling_file_mode = "a";
+static int profiling_rule_json = 0;
 
 /**
  * Sort orders for dumping profiled rules.
@@ -182,6 +179,13 @@ void SCProfilingRulesGlobalInit(void)
 
                 profiling_output_to_file = 1;
             }
+            if (ConfNodeChildValueIsTrue(conf, "json")) {
+#ifdef HAVE_LIBJANSSON
+                profiling_rule_json = 1;
+#else
+                SCLogWarning(SC_ERR_NO_JSON_SUPPORT, "no json support compiled in, using plain output");
+#endif
+            }
         }
     }
 }
@@ -284,6 +288,134 @@ SCProfileSummarySortByMaxTicks(const void *a, const void *b)
         return s0->max > s1->max ? -1 : 1;
 }
 
+#ifdef HAVE_LIBJANSSON
+
+static void DumpJson(FILE *fp, SCProfileSummary *summary, uint32_t count, uint64_t total_ticks)
+{
+    char timebuf[64];
+    uint32_t i;
+    struct timeval tval;
+
+    json_t *js = json_object();
+    if (js == NULL)
+        return;
+    json_t *jsa = json_array();
+    if (jsa == NULL) {
+        json_decref(js);
+        return;
+    }
+
+    gettimeofday(&tval, NULL);
+    CreateIsoTimeString(&tval, timebuf, sizeof(timebuf));
+    json_object_set_new(js, "timestamp", json_string(timebuf));
+
+    for (i = 0; i < count; i++) {
+        /* Stop dumping when we hit our first rule with 0 checks.  Due
+         * to sorting this will be the beginning of all the rules with
+         * 0 checks. */
+        if (summary[i].checks == 0)
+            break;
+
+        json_t *jsm = json_object();
+        if (jsm) {
+            json_object_set_new(jsm, "signature_id", json_integer(summary[i].sid));
+            json_object_set_new(jsm, "gid", json_integer(summary[i].gid));
+            json_object_set_new(jsm, "rev", json_integer(summary[i].rev));
+
+            json_object_set_new(jsm, "checks", json_integer(summary[i].checks));
+            json_object_set_new(jsm, "matches", json_integer(summary[i].matches));
+
+            json_object_set_new(jsm, "ticks_total", json_integer(summary[i].ticks));
+            json_object_set_new(jsm, "ticks_max", json_integer(summary[i].max));
+            json_object_set_new(jsm, "ticks_avg", json_integer(summary[i].avgticks));
+            json_object_set_new(jsm, "ticks_avg_match", json_integer(summary[i].avgticks_match));
+            json_object_set_new(jsm, "ticks_avg_nomatch", json_integer(summary[i].avgticks_no_match));
+
+            double percent = (long double)summary[i].ticks /
+                (long double)total_ticks * 100;
+            json_object_set_new(jsm, "percent", json_integer(percent));
+            json_array_append(jsa, jsm);
+        }
+    }
+    json_object_set_new(js, "rules", jsa);
+
+    char *js_s = json_dumps(js,
+            JSON_PRESERVE_ORDER|JSON_COMPACT|JSON_ENSURE_ASCII|
+#ifdef JSON_ESCAPE_SLASH
+            JSON_ESCAPE_SLASH
+#else
+            0
+#endif
+            );
+
+    if (unlikely(js_s == NULL))
+        return;
+    fprintf(fp, "%s", js_s);
+    free(js_s);
+    json_decref(js);
+}
+
+#endif /* HAVE_LIBJANSSON */
+
+static void DumpText(FILE *fp, SCProfileSummary *summary, uint32_t count, uint64_t total_ticks)
+{
+    uint32_t i;
+    struct timeval tval;
+    struct tm *tms;
+    gettimeofday(&tval, NULL);
+    struct tm local_tm;
+    tms = SCLocalTime(tval.tv_sec, &local_tm);
+
+    fprintf(fp, "  ----------------------------------------------"
+            "----------------------------\n");
+    fprintf(fp, "  Date: %" PRId32 "/%" PRId32 "/%04d -- "
+            "%02d:%02d:%02d\n", tms->tm_mon + 1, tms->tm_mday, tms->tm_year + 1900,
+            tms->tm_hour,tms->tm_min, tms->tm_sec);
+    fprintf(fp, "  ----------------------------------------------"
+            "----------------------------\n");
+    fprintf(fp, "   %-8s %-12s %-8s %-8s %-12s %-6s %-8s %-8s %-11s %-11s %-11s %-11s\n", "Num", "Rule", "Gid", "Rev", "Ticks", "%", "Checks", "Matches", "Max Ticks", "Avg Ticks", "Avg Match", "Avg No Match");
+    fprintf(fp, "  -------- "
+        "------------ "
+        "-------- "
+        "-------- "
+        "------------ "
+        "------ "
+        "-------- "
+        "-------- "
+        "----------- "
+        "----------- "
+        "----------- "
+        "-------------- "
+        "\n");
+    for (i = 0; i < MIN(count, profiling_rules_limit); i++) {
+
+        /* Stop dumping when we hit our first rule with 0 checks.  Due
+         * to sorting this will be the beginning of all the rules with
+         * 0 checks. */
+        if (summary[i].checks == 0)
+            break;
+
+        double percent = (long double)summary[i].ticks /
+            (long double)total_ticks * 100;
+        fprintf(fp,
+            "  %-8"PRIu32" %-12u %-8"PRIu32" %-8"PRIu32" %-12"PRIu64" %-6.2f %-8"PRIu64" %-8"PRIu64" %-11"PRIu64" %-11.2f %-11.2f %-11.2f\n",
+            i + 1,
+            summary[i].sid,
+            summary[i].gid,
+            summary[i].rev,
+            summary[i].ticks,
+            percent,
+            summary[i].checks,
+            summary[i].matches,
+            summary[i].max,
+            summary[i].avgticks,
+            summary[i].avgticks_match,
+            summary[i].avgticks_no_match);
+    }
+
+    fprintf(fp,"\n");
+}
+
 /**
  * \brief Dump rule profiling information to file
  *
@@ -298,8 +430,6 @@ SCProfilingRuleDump(SCProfileDetectCtx *rules_ctx)
     if (rules_ctx == NULL)
         return;
 
-    struct timeval tval;
-    struct tm *tms;
     if (profiling_output_to_file == 1) {
         fp = fopen(profiling_file_name, profiling_file_mode);
 
@@ -383,59 +513,15 @@ SCProfilingRuleDump(SCProfileDetectCtx *rules_ctx)
                     SCProfileSummarySortByAvgTicksNoMatch);
             break;
     }
-
-    gettimeofday(&tval, NULL);
-    struct tm local_tm;
-    tms = SCLocalTime(tval.tv_sec, &local_tm);
-
-    fprintf(fp, "  ----------------------------------------------"
-            "----------------------------\n");
-    fprintf(fp, "  Date: %" PRId32 "/%" PRId32 "/%04d -- "
-            "%02d:%02d:%02d\n", tms->tm_mon + 1, tms->tm_mday, tms->tm_year + 1900,
-            tms->tm_hour,tms->tm_min, tms->tm_sec);
-    fprintf(fp, "  ----------------------------------------------"
-            "----------------------------\n");
-    fprintf(fp, "   %-8s %-12s %-8s %-8s %-12s %-6s %-8s %-8s %-11s %-11s %-11s %-11s\n", "Num", "Rule", "Gid", "Rev", "Ticks", "%", "Checks", "Matches", "Max Ticks", "Avg Ticks", "Avg Match", "Avg No Match");
-    fprintf(fp, "  -------- "
-        "------------ "
-        "-------- "
-        "-------- "
-        "------------ "
-        "------ "
-        "-------- "
-        "-------- "
-        "----------- "
-        "----------- "
-        "----------- "
-        "-------------- "
-        "\n");
-    for (i = 0; i < MIN(count, profiling_rules_limit); i++) {
-
-        /* Stop dumping when we hit our first rule with 0 checks.  Due
-         * to sorting this will be the beginning of all the rules with
-         * 0 checks. */
-        if (summary[i].checks == 0)
-            break;
-
-        double percent = (long double)summary[i].ticks /
-            (long double)total_ticks * 100;
-        fprintf(fp,
-            "  %-8"PRIu32" %-12u %-8"PRIu32" %-8"PRIu32" %-12"PRIu64" %-6.2f %-8"PRIu64" %-8"PRIu64" %-11"PRIu64" %-11.2f %-11.2f %-11.2f\n",
-            i + 1,
-            summary[i].sid,
-            summary[i].gid,
-            summary[i].rev,
-            summary[i].ticks,
-            percent,
-            summary[i].checks,
-            summary[i].matches,
-            summary[i].max,
-            summary[i].avgticks,
-            summary[i].avgticks_match,
-            summary[i].avgticks_no_match);
+#ifdef HAVE_LIBJANSSON
+    if (profiling_rule_json) {
+        DumpJson(fp, summary, count, total_ticks);
+    } else
+#endif
+    {
+        DumpText(fp, summary, count, total_ticks);
     }
 
-    fprintf(fp,"\n");
     if (fp != stdout)
         fclose(fp);
     SCFree(summary);
diff --git a/framework/src/suricata/src/util-runmodes.c b/framework/src/suricata/src/util-runmodes.c
index c9b61e6a..a327a512 100644
--- a/framework/src/suricata/src/util-runmodes.c
+++ b/framework/src/suricata/src/util-runmodes.c
@@ -420,7 +420,7 @@ int RunModeSetLiveCaptureWorkers(ConfigIfaceParserFunc ConfigParser,
 
     for (ldev = 0; ldev < nlive; ldev++) {
         char *live_dev_c = NULL;
-        if (live_dev != NULL) {
+        if ((nlive <= 1) && (live_dev != NULL)) {
             aconf = ConfigParser(live_dev);
             live_dev_c = SCStrdup(live_dev);
             if (unlikely(live_dev_c == NULL)) {
diff --git a/framework/src/suricata/src/util-validate.h b/framework/src/suricata/src/util-validate.h
index b5c3da26..b7647a7d 100644
--- a/framework/src/suricata/src/util-validate.h
+++ b/framework/src/suricata/src/util-validate.h
@@ -96,11 +96,14 @@
     }                                               \
 } while(0)
 
+#define DEBUG_VALIDATE_BUG_ON(exp) BUG_ON((exp))
+
 #else /* DEBUG_VALIDATE */
 
 #define DEBUG_ASSERT_FLOW_LOCKED(f)
 #define DEBUG_VALIDATE_FLOW(f)
 #define DEBUG_VALIDATE_PACKET(p)
+#define DEBUG_VALIDATE_BUG_ON(exp)
 
 #endif /* DEBUG_VALIDATE */
 
diff --git a/framework/src/suricata/suricata.yaml.in b/framework/src/suricata/suricata.yaml.in
index 56d4d362..af54b527 100644
--- a/framework/src/suricata/suricata.yaml.in
+++ b/framework/src/suricata/suricata.yaml.in
@@ -44,9 +44,13 @@ host-mode: auto
 #  user: suri
 #  group: suri
 
+# Some logging module will use that name in event as identifier. The default
+# value is the hostname
+#sensor-name: suricata
+
 # Default pid file.
 # Will use this file if no --pidfile in command options.
-#pid-file: /var/run/suricata.pid
+#pid-file: @e_rundir@suricata.pid
 
 # Daemon working directory
 # Suricata will change directory to this one if provided
@@ -92,7 +96,7 @@ outputs:
   # Extensible Event Format (nicknamed EVE) event log in JSON format
   - eve-log:
       enabled: yes
-      filetype: regular #regular|syslog|unix_dgram|unix_stream
+      filetype: regular #regular|syslog|unix_dgram|unix_stream|redis
       filename: eve.json
       #prefix: "@cee: " # prefix to prepend to each log entry
       # the following are valid when type: syslog above
@@ -100,6 +104,18 @@ outputs:
       #facility: local5
       #level: Info ## possible levels: Emergency, Alert, Critical,
                    ## Error, Warning, Notice, Info, Debug
+      #redis:
+      #  server: 127.0.0.1
+      #  port: 6379
+      #  mode: list ## possible values: list (default), channel
+      #  key: suricata ## key or channel to use (default to suricata)
+      # Redis pipelining set up. This will enable to only do a query every
+      # 'batch-size' events. This should lower the latency induced by network
+      # connection at the cost of some memory. There is no flushing implemented
+      # so this setting as to be reserved to high traffic suricata.
+      #  pipelining:
+      #    enabled: yes ## set enable to yes to enable query pipelining
+      #    batch-size: 10 ## number of entry to keep in buffer
       types:
         - alert:
             # payload: yes           # enable dumping payload in Base64
@@ -108,6 +124,7 @@ outputs:
             # http: yes              # enable dumping of http fields
             # tls: yes               # enable dumping of tls fields
             # ssh: yes               # enable dumping of ssh fields
+            # smtp: yes              # enable dumping of smtp fields
 
             # HTTP X-Forwarded-For support by adding an extra field or overwriting
             # the source or destination IP address (depending on flow direction)
@@ -139,7 +156,19 @@ outputs:
             force-md5: no     # force logging of md5 checksums
         #- drop:
         #    alerts: no       # log alerts that caused drops
-        - smtp
+        - smtp:
+            #extended: yes # enable this for extended logging information
+            # this includes: bcc, message-id, subject, x_mailer, user-agent
+            # custom fields logging from the list:
+            #  reply-to, bcc, message-id, subject, x-mailer, user-agent, received,
+            #  x-originating-ip, in-reply-to, references, importance, priority,
+            #  sensitivity, organization, content-md5, date
+            #custom: [received, x-mailer, x-originating-ip, relays, reply-to, bcc]
+            # output md5 of fields: body, subject
+            # for the body you need to set app-layer.protocols.smtp.mime.body-md5
+            # to yes
+            #md5: [body, subject]
+
         - ssh
         - stats:
             totals: yes       # stats for all threads merged together
@@ -162,6 +191,10 @@ outputs:
       # Sensor ID field of unified2 alerts.
       #sensor-id: 0
 
+      # Include payload of packets related to alerts. Defaults to true, set to
+      # false if payload is not required.
+      #payload: yes
+
       # HTTP X-Forwarded-For support by adding the unified2 extra header or
       # overwriting the source or destination IP address (depending on flow
       # direction) with the one reported in the X-Forwarded-For HTTP header.
@@ -947,7 +980,7 @@ logging:
       # type: json
   - file:
       enabled: no
-      filename: /var/log/suricata.log
+      filename: @e_logdir@suricata.log
       # type: json
   - syslog:
       enabled: no
@@ -1291,6 +1324,9 @@ app-layer:
 
         # Extract URLs and save in state data structure
         extract-urls: yes
+        # Set to yes to compute the md5 of the mail body. You will then
+        # be able to journalize it.
+        body-md5: no
       # Configure inspected-tracker for file_data keyword
       inspected-tracker:
         content-limit: 1000
@@ -1475,9 +1511,12 @@ profiling:
     # Sort options: ticks, avgticks, checks, matches, maxticks
     sort: avgticks
 
-    # Limit the number of items printed at exit.
+    # Limit the number of items printed at exit (ignored for json).
     limit: 100
 
+    # output to json
+    json: true
+
   # per keyword profiling
   keywords:
     enabled: yes