summaryrefslogtreecommitdiffstats
path: root/rubbos/app/tomcat-connectors-1.2.32-src/tools/reports
diff options
context:
space:
mode:
authorhongbotian <hongbo.tianhongbo@huawei.com>2015-11-30 02:41:33 -0500
committerhongbotian <hongbo.tianhongbo@huawei.com>2015-11-30 02:43:36 -0500
commit9401f816dd0d9d550fe98a8507224bde51c4b847 (patch)
tree94f2d7a7893a787bafdca8b5ef063ea316938874 /rubbos/app/tomcat-connectors-1.2.32-src/tools/reports
parente8ec7aa8e38a93f5b034ac74cebce5de23710317 (diff)
upload tomcat
JIRA: BOTTLENECK-7 Change-Id: I875d474869efd76ca203c30b60ebc0c3ee606d0e Signed-off-by: hongbotian <hongbo.tianhongbo@huawei.com>
Diffstat (limited to 'rubbos/app/tomcat-connectors-1.2.32-src/tools/reports')
-rw-r--r--rubbos/app/tomcat-connectors-1.2.32-src/tools/reports/README.txt33
-rwxr-xr-xrubbos/app/tomcat-connectors-1.2.32-src/tools/reports/tomcat_reports.pl431
-rwxr-xr-xrubbos/app/tomcat-connectors-1.2.32-src/tools/reports/tomcat_trend.pl408
3 files changed, 872 insertions, 0 deletions
diff --git a/rubbos/app/tomcat-connectors-1.2.32-src/tools/reports/README.txt b/rubbos/app/tomcat-connectors-1.2.32-src/tools/reports/README.txt
new file mode 100644
index 00000000..898a9df5
--- /dev/null
+++ b/rubbos/app/tomcat-connectors-1.2.32-src/tools/reports/README.txt
@@ -0,0 +1,33 @@
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+This directory contains perl scripts which can be used to generate
+statistics for tomcat requests and errors logged by mod_jk.
+
+See the comments in the scripts for more details.
+
+A great deal of statistical data is generated but at this time
+only long term trend graphs are being created and no reports.
+This is only a start. Many more graphs and reports could be
+generated from the data. Please consider contributing back any
+new reports or graphs you create. Thanks.
+
+Requires the following perl modules and libraries:
+
+GD 1.8.x graphics library http://www.boutell.com/gd/.
+GD 1.4.x perl module.
+GD Graph perl module.
+GD TextUtil perl module.
+StatisticsDescriptive perl module.
diff --git a/rubbos/app/tomcat-connectors-1.2.32-src/tools/reports/tomcat_reports.pl b/rubbos/app/tomcat-connectors-1.2.32-src/tools/reports/tomcat_reports.pl
new file mode 100755
index 00000000..4995c251
--- /dev/null
+++ b/rubbos/app/tomcat-connectors-1.2.32-src/tools/reports/tomcat_reports.pl
@@ -0,0 +1,431 @@
+#!/usr/local/bin/perl
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# $Id: tomcat_reports.pl 466585 2006-10-21 22:16:34Z markt $
+
+# Author: Glenn Nielsen
+
+# Script for generating reports and graphs using statistical data generated
+# by the tomcat_trend.pl script.
+#
+# The following graphs are created:
+#
+# tomcat_request.png
+# Long term trend graph of total number of tomcat requests handled
+#
+# tomcat_median.png
+# Long term overall trend graph of tomcat request latency median
+#
+# tomcat_deviation.png
+# Long term overall trend graph of tomcat request mean and standard deviation
+#
+# tomcat_error.png
+# Long term trend graph of requests rejected by tomcat. Shows requests rejected
+# when tomcat has no request processors available. Can be an indicator that tomcat
+# is overloaded or having other scaling problems.
+#
+# tomcat_client.png
+# Long term trend graph of requests forward to tomcat which were aborted by the remote
+# client (browser). You will normally see some aborted requests. High numbers of these
+# can be an indicator that tomcat is overloaded or there are requests which have very high
+# latency.
+#
+# tomcat_reports.pl <directory where statistics are archived> <directory to place graphs/reports in>
+
+use GD;
+use GD::Graph;
+use GD::Graph::Data;
+use GD::Graph::lines;
+use GD::Graph::linespoints;
+use Statistics::Descriptive;
+use Time::Local;
+
+# Constants
+
+%MON = ('JAN' => 0, 'Jan' => 0,
+ 'FEB' => 1, 'Feb' => 1,
+ 'MAR' => 2, 'Mar' => 2,
+ 'APR' => 3, 'Apr' => 3,
+ 'MAY' => 4, 'May' => 4,
+ 'JUN' => 5, 'Jun' => 5,
+ 'JUL' => 6, 'Jul' => 6,
+ 'AUG' => 7, 'Aug' => 7,
+ 'SEP' => 8, 'Sep' => 8,
+ 'OCT' => 9, 'Oct' => 9,
+ 'NOV' => 10, 'Nov' => 10,
+ 'DEC' => 11, 'Dec' => 11,);
+
+@Months = ("Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec");
+
+# Check the args
+
+$archivedir = $ARGV[0];
+$reportdir = $ARGV[1];
+
+die "Usage: $0 archivedir reportdir"
+ unless( length($archivedir) && ($reportdir) );
+
+die "Archive Directory $archivedir doesn't exist"
+ unless( -d $archivedir);
+
+die "Report Directory $reportdir doesn't exist"
+ unless( -d $reportdir);
+
+# Read in data from file
+die "Archive Directory $archivedir has no global.data file"
+ unless( -e "$archivedir/global.data" );
+
+@Data = `tail -365 $archivedir/global.data`;
+$numdays = $#Data;
+$daycounter = $numdays;
+
+foreach( @Data ) {
+ $line = $_;
+ chomp($line);
+ ($date,$count,$median,$mean,$stddev,$min,$max,$client_gone,$tomcat_full) = split /\s+/,$line;
+ # print "$daycounter $date $count $median $mean $stdev $min $max $client_gone $tomcat_full\n";
+ $start_time = $date unless $start_time>0;
+ $end_time = $date;
+ push @days,int($daycounter/7);
+ push @count,$count;
+ push @median,$median;
+ push @mean,$mean;
+ push @stddev,$mean+$stddev;
+ push @min,$min;
+ push @max,$max;
+ push @client_gone,$client_gone;
+ push @tomcat_full,$tomcat_full;
+ $daycounter--;
+}
+
+($day,$mon,$year) = (localtime($start_time))[3..5];
+$year += 1900;
+$startdate = "$Months[$mon] $day, $year";
+($day,$mon,$year) = (localtime($end_time))[3..5];
+$year += 1900;
+$enddate = "$Months[$mon] $day, $year";
+
+# Output request trend graph
+$outfile = "$reportdir/tomcat_request.png";
+unlink $outfile;
+
+$stats = Statistics::Descriptive::Sparse->new();
+$stats->add_data(@count);
+$max = $stats->max();
+$min = $stats->min();
+
+&RequestGraph();
+
+# Output median latency trend graph
+$outfile = "$reportdir/tomcat_median.png";
+unlink $outfile;
+
+$stats = Statistics::Descriptive::Sparse->new();
+$stats->add_data(@median);
+$max = $stats->max();
+$min = $stats->min();
+
+&MedianGraph();
+
+# Output latency deviation trend graph
+$outfile = "$reportdir/tomcat_deviation.png";
+unlink $outfile;
+
+$stats = Statistics::Descriptive::Sparse->new();
+$stats->add_data(@stddev);
+$stats->add_data(@mean);
+$max = $stats->max();
+$min = $stats->min();
+
+&DeviationGraph();
+
+# Output request error trend graph
+$outfile = "$reportdir/tomcat_error.png";
+unlink $outfile;
+
+$stats = Statistics::Descriptive::Sparse->new();
+$stats->add_data(@tomcat_full);
+$max = $stats->max();
+$min = $stats->min();
+
+&ErrorGraph();
+
+# Output request error trend graph
+$outfile = "$reportdir/tomcat_client.png";
+unlink $outfile;
+
+$stats = Statistics::Descriptive::Sparse->new();
+$stats->add_data(@client_gone);
+$max = $stats->max();
+$min = $stats->min();
+
+&ClientGraph();
+
+exit;
+
+sub RequestGraph {
+
+ $graph = GD::Graph::lines->new(800,600);
+ @data = (\@days,\@count);
+
+ $div = 100;
+ $div = 500 if $max >= 2000;
+ $div = 1000 if $max >= 5000;
+ $div = 5000 if $max >= 20000;
+ $div = 10000 if $max >= 50000;
+ $div = 50000 if $max >= 200000;
+ $div = 100000 if $max >= 500000;
+ $div = 500000 if $max >= 2000000;
+ $div = 1000000 if $max >= 5000000;
+ $ymax = (int($max/$div) + 1)*$div;
+ $ymin = int($min/$div)*$div;
+ $ytick = ($ymax - $ymin)/$div;
+
+ $graph->set(
+ y_label => 'Requests',
+ title => "Tomcat Requests by Day from $startdate to $enddate",
+ y_min_value => $ymin,
+ y_max_value => $ymax,
+ y_tick_number => $ytick,
+ y_number_format => \&val_format,
+ x_label => 'Weeks Ago',
+ x_label_skip => 7,
+ x_tick_offset => $numdays%7,
+ dclrs => [ qw(green) ],
+ legend_placement => 'BC'
+ ) or warn $graph->error;
+
+ $graph->set_legend( 'Requests' );
+ $graph->set_title_font(GD::gdGiantFont);
+ $graph->set_x_axis_font(GD::gdSmallFont);
+ $graph->set_y_axis_font(GD::gdSmallFont);
+ $graph->set_legend_font(GD::gdSmallFont);
+ $gd = $graph->plot(\@data);
+ die "Graph Plot Failed: " . $graph->error unless defined $gd;
+
+ open IMG, ">$outfile" or die $!;
+ print IMG $gd->png or die $gd->error;
+ close IMG;
+}
+
+sub MedianGraph {
+
+ $graph = GD::Graph::lines->new(800,600);
+ @data = (\@days,\@median);
+
+ $div = .05;
+ $div = .1 if $max >= .5;
+ $div = .5 if $max >= 2;
+ $div = 1 if $max >= 5;
+ $div = 5 if $max >= 20;
+ $div = 10 if $max >= 50;
+ $div = 50 if $max >= 200;
+ $div = 100 if $max >= 500;
+ $ymax = (int($max/$div) + 1)*$div;
+ $ytick = $ymax/$div;
+
+ $graph->set(
+ y_label => 'Latency (Seconds)',
+ title => "Tomcat Request Median Latency by Day from $startdate to $enddate",
+ y_min_value => 0,
+ y_max_value => $ymax,
+ y_tick_number => $ytick,
+ y_number_format => \&val_format,
+ x_label => 'Weeks Ago',
+ x_label_skip => 7,
+ x_tick_offset => $numdays%7,
+ dclrs => [ qw(green) ],
+ legend_placement => 'BC'
+ ) or warn $graph->error;
+
+ $graph->set_legend( 'Median' );
+ $graph->set_title_font(GD::gdGiantFont);
+ $graph->set_x_axis_font(GD::gdSmallFont);
+ $graph->set_y_axis_font(GD::gdSmallFont);
+ $graph->set_legend_font(GD::gdSmallFont);
+ $gd = $graph->plot(\@data);
+ die "Graph Plot Failed: " . $graph->error unless defined $gd;
+
+ open IMG, ">$outfile" or die $!;
+ print IMG $gd->png or die $gd->error;
+ close IMG;
+}
+
+sub DeviationGraph {
+
+ $graph = GD::Graph::lines->new(800,600);
+ @data = (\@days,\@mean,\@stddev);
+
+ $div = .1;
+ $div = .5 if $max >= 2;
+ $div = 1 if $max >= 5;
+ $div = 5 if $max >= 20;
+ $div = 10 if $max >= 50;
+ $div = 50 if $max >= 200;
+ $div = 100 if $max >= 500;
+ $ymax = (int($max/$div) + 1)*$div;
+ $ytick = $ymax/$div;
+
+ $graph->set(
+ y_label => 'Latency (Seconds)',
+ title => "Tomcat Request Latency Mean and Deviation by Day from $startdate to $enddate",
+ y_max_value => $ymax,
+ y_tick_number => $ytick,
+ x_label => 'Weeks Ago',
+ x_label_skip => 7,
+ x_tick_offset => $numdays%7,
+ dclrs => [ qw(green yellow) ],
+ legend_placement => 'BC'
+ ) or warn $graph->error;
+
+ $graph->set_legend( 'Mean', 'Mean plus Standard Deviation' );
+ $graph->set_title_font(GD::gdGiantFont);
+ $graph->set_x_axis_font(GD::gdSmallFont);
+ $graph->set_y_axis_font(GD::gdSmallFont);
+ $graph->set_legend_font(GD::gdSmallFont);
+ $gd = $graph->plot(\@data);
+ die "Graph Plot Failed: " . $graph->error unless defined $gd;
+
+ open IMG, ">$outfile" or die $!;
+ print IMG $gd->png or die $gd->error;
+ close IMG;
+}
+
+sub ErrorGraph {
+
+ $graph = GD::Graph::lines->new(800,600);
+ @data = (\@days,\@tomcat_full);
+
+ $div = 5;
+ $div = 10 if $max >=100;
+ $div = 50 if $max >= 200;
+ $div = 100 if $max >= 1000;
+ $div = 500 if $max >= 2000;
+ $div = 1000 if $max >= 5000;
+ $div = 5000 if $max >= 20000;
+ $div = 10000 if $max >= 50000;
+ $div = 50000 if $max >= 200000;
+ $div = 100000 if $max >= 500000;
+ $div = 500000 if $max >= 2000000;
+ $div = 1000000 if $max >= 5000000;
+ $ymax = (int($max/$div) + 1)*$div;
+ $ymin = int($min/$div)*$div;
+ $ytick = ($ymax - $ymin)/$div;
+
+ $graph->set(
+ y_label => 'Requests',
+ title => "Tomcat Rejected Request by Day from $startdate to $enddate",
+ y_min_value => $ymin,
+ y_max_value => $ymax,
+ y_tick_number => $ytick,
+ y_number_format => \&val_format,
+ x_label => 'Weeks Ago',
+ x_label_skip => 7,
+ x_tick_offset => $numdays%7,
+ dclrs => [ qw(green) ],
+ legend_placement => 'BC'
+ ) or warn $graph->error;
+
+ $graph->set_legend( 'Tomcat Rejected Requests' );
+ $graph->set_title_font(GD::gdGiantFont);
+ $graph->set_x_axis_font(GD::gdSmallFont);
+ $graph->set_y_axis_font(GD::gdSmallFont);
+ $graph->set_legend_font(GD::gdSmallFont);
+ $gd = $graph->plot(\@data);
+ die "Graph Plot Failed: " . $graph->error unless defined $gd;
+
+ open IMG, ">$outfile" or die $!;
+ print IMG $gd->png or die $gd->error;
+ close IMG;
+}
+
+sub ClientGraph {
+
+ $graph = GD::Graph::lines->new(800,600);
+ @data = (\@days,\@client_gone);
+
+ $div = 5;
+ $div = 10 if $max >=100;
+ $div = 50 if $max >= 200;
+ $div = 100 if $max >= 1000;
+ $div = 500 if $max >= 2000;
+ $div = 1000 if $max >= 5000;
+ $div = 5000 if $max >= 20000;
+ $div = 10000 if $max >= 50000;
+ $div = 50000 if $max >= 200000;
+ $div = 100000 if $max >= 500000;
+ $div = 500000 if $max >= 2000000;
+ $div = 1000000 if $max >= 5000000;
+ $ymax = (int($max/$div) + 1)*$div;
+ $ymin = int($min/$div)*$div;
+ $ytick = ($ymax - $ymin)/$div;
+
+ $graph->set(
+ y_label => 'Requests',
+ title => "Tomcat Client Aborted Requests by Day from $startdate to $enddate",
+ y_min_value => $ymin,
+ y_max_value => $ymax,
+ y_tick_number => $ytick,
+ y_number_format => \&val_format,
+ x_label => 'Weeks Ago',
+ x_label_skip => 7,
+ x_tick_offset => $numdays%7,
+ dclrs => [ qw(green) ],
+ legend_placement => 'BC'
+ ) or warn $graph->error;
+
+ $graph->set_legend( 'Tomcat Client Aborted Requests' );
+ $graph->set_title_font(GD::gdGiantFont);
+ $graph->set_x_axis_font(GD::gdSmallFont);
+ $graph->set_y_axis_font(GD::gdSmallFont);
+ $graph->set_legend_font(GD::gdSmallFont);
+ $gd = $graph->plot(\@data);
+ die "Graph Plot Failed: " . $graph->error unless defined $gd;
+
+ open IMG, ">$outfile" or die $!;
+ print IMG $gd->png or die $gd->error;
+ close IMG;
+}
+
+sub val_format {
+ my $value = shift;
+ my $ret;
+
+ $ret = $value;
+ if( $ret =~ /\./ ) {
+ $ret =~ s/\.(\d\d\d).*/\.$1/;
+ } else {
+ $ret =~ s/(\d+)(\d\d\d)$/$1,$2/;
+ $ret =~ s/(\d+)(\d\d\d),(\d\d\d)$/$1,$2,$3/;
+ }
+ return $ret;
+}
+
+sub size_format {
+ my $value = shift;
+ my $ret;
+
+ if( $max >= 5000 ) {
+ $value = int(($value/1024)+.5);
+ }
+ $ret = $value;
+ $ret =~ s/(\d+)(\d\d\d)$/$1,$2/;
+ $ret =~ s/(\d+)(\d\d\d),(\d\d\d)$/$1,$2,$3/;
+ return $ret;
+}
diff --git a/rubbos/app/tomcat-connectors-1.2.32-src/tools/reports/tomcat_trend.pl b/rubbos/app/tomcat-connectors-1.2.32-src/tools/reports/tomcat_trend.pl
new file mode 100755
index 00000000..ee933a2a
--- /dev/null
+++ b/rubbos/app/tomcat-connectors-1.2.32-src/tools/reports/tomcat_trend.pl
@@ -0,0 +1,408 @@
+#!/usr/local/bin/perl
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# $Id: tomcat_trend.pl 466585 2006-10-21 22:16:34Z markt $
+
+# Author: Glenn Nielsen
+
+# Script for analyzing mod_jk.log data when logging tomcat request data using
+# the JkRequestLogFormat Apache mod_jk configuration.
+#
+# Generates statistics for request latency and errors. Archives the generated
+# data to files for later use in long term trend graphs and reports.
+#
+# tomcat_trend.pl <directory containing mod_jk.log> <directory for archiving statistics>
+
+use FileHandle;
+use Statistics::Descriptive;
+use Time::Local;
+
+# Constants
+
+%MON = ('JAN' => 0, 'Jan' => 0,
+ 'FEB' => 1, 'Feb' => 1,
+ 'MAR' => 2, 'Mar' => 2,
+ 'APR' => 3, 'Apr' => 3,
+ 'MAY' => 4, 'May' => 4,
+ 'JUN' => 5, 'Jun' => 5,
+ 'JUL' => 6, 'Jul' => 6,
+ 'AUG' => 7, 'Aug' => 7,
+ 'SEP' => 8, 'Sep' => 8,
+ 'OCT' => 9, 'Oct' => 9,
+ 'NOV' => 10, 'Nov' => 10,
+ 'DEC' => 11, 'Dec' => 11,);
+
+@Months = ("Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec");
+
+# Check the args
+
+$logdir= $ARGV[0];
+$archivedir = $ARGV[1];
+
+die "Usage: $0 logdir archivedir"
+ unless( length($logdir) && length($archivedir) );
+
+die "Log Directory $logdir doesn't exist"
+ unless( -d $logdir);
+
+die "Archive Directory $archivedir doesn't exist"
+ unless( -d $archivedir);
+
+# Get start date from global.data if it exists
+
+if( -e "$archivedir/global.data" ) {
+ # Get the start date from the last entry in global.data
+ @tail = `tail -1 $archivedir/global.data`;
+ $startdate = (split /\s+/,$tail[0])[0];
+ ($day, $mon, $year) = (localtime($startdate))[3..5];
+ if ($day == 31) {
+ $day=1;
+ $month++;
+ if ($month > 11) {
+ $month=0;
+ $year++;
+ }
+ }
+ $startdate = timelocal(0,0,0,$day+1,$mon,$year);
+
+}
+
+($day, $mon, $year) = (localtime(time))[3..5];
+$curdate = timelocal(0,0,0,$day,$mon,$year);
+print "Today: " . scalar(localtime($curdate)) . "\n";
+
+# Get the log files names and date they start
+@logs = `ls -1 $logdir/mod_jk.log*`;
+foreach( @logs ) {
+ $logfile = $_;
+ chomp($logfile);
+ next if ( $logfile =~ /\.(bz2|gz|zip)$/ );
+ @head = `head -1 $logfile`;
+ ($mon, $day, $time, $year) = (split /\s+/,$head[0])[1..4];
+ ($hour, $min, $sec) = split /:/,$time;
+ $year =~ s/\]$//;
+ $logtime = timelocal($sec,$min,$hour,$day,$MON{$mon},$year-1900);
+ $modjklog{$logtime} = $logfile;
+}
+
+# Set the startdate if this is the first time processing the logs
+# If we have a startdate, remove log files we con't need to process
+foreach $logtime ( sort {$a <=> $b} keys %modjklog ) {
+ # If logs haven't been processed before, set startdate to time of
+ # first log entry
+ if( $startdate !~ /^\d+$/ ) {
+ $startdate = $logtime;
+ ($day, $mon, $year) = (localtime($startdate))[3..5];
+ $startdate = timelocal(0,0,0,$day,$mon,$year);
+ last;
+ }
+ if( $logtime > $startdate ) {
+ last;
+ }
+ # Save the previous log file since start date may start here
+ $prevlogfile = $modjklog{$logtime};
+ $prevlogtime = $logtime;
+ # Remove log files we don't need to process
+ delete $modjklog{$logtime};
+}
+
+# Add back in the previous log file where we need to start processing
+if( defined $prevlogtime ) {
+ $modjklog{$prevlogtime} = $prevlogfile;
+}
+
+print "StartDate: " . scalar(localtime($startdate)) . "\n";
+$processdate = $startdate;
+
+foreach $key ( sort {$a <=> $b} keys %modjklog ) {
+ $logtime = $processdate;
+ $logfile = $modjklog{$key};
+ print "Processing log: $logfile\n";
+ last if( $key >= $curdate );
+ $fh = new FileHandle "<$logfile";
+ die "Open of logfile $logfile failed: $!"
+ unless defined $fh;
+ while( $line = $fh->getline) {
+ chomp($line);
+ ($mon, $day, $time, $year) = (split /\s+/,$line)[1..4];
+ ($hour, $min, $sec) = split /:/,$time;
+ $year =~ s/\]$//;
+ if( $day !~ /^\d+/ || $hour !~ /^\d+/ || $min!~ /^\d+/ || $sec !~ /^\d+/ ) {
+ print "Unknown log entry: $origline\n" unless $origline =~ /\.c /;
+ next;
+ }
+ $logtime = timelocal($sec,$min,$hour,$day,$MON{$mon},$year-1900);
+
+ if( $logtime > $processdate ) {
+ $origline = $line;
+ # Strip off the leading date and time
+ $line =~ s/^\[.*\] //;
+
+ # See if this is a new 5 minute period
+ $interval = int($logtime/300);
+ if( $interval != $previnterval ) {
+ if( defined $previnterval ) {
+ &IntervalStats(\%Global,\%Interval,$previnterval*300);
+ }
+ undef %Interval;
+ undef @IntervalLatency;
+ undef %IntervalWorkers;
+ $Interval{tomcat_full} = 0;
+ $Interval{client_gone} = 0;
+ $Interval{latency} = \@IntervalLatency;
+ $Interval{workers} = \%IntervalWorkers;
+ $previnterval = $interval;
+ }
+
+ # See if this is a new day
+ if( $day != $prevday ) {
+ if( defined $prevday ) {
+ &DailyStats($processdate,\%Global);
+ }
+ undef %Global;
+ undef %GlobalWorkers;
+ undef @GlobalLatency;
+ $Global{tomcat_full} = 0;
+ $Global{client_gone} = 0;
+ $Global{interval} = "";
+ $Global{latency} = \@GlobalLatency;
+ $Global{workers} = \%GlobalWorkers;
+ $Global{errors} = "";
+ $prevday = $day;
+ $processdate = $logtime;
+ }
+
+ # Stop processing if logtime is today
+ last if( $logtime >= $curdate );
+
+ if( $line =~ /\d\)\]{0,1}: / ) {
+ # Handle a mod_jk error
+ if( $line =~ /(jk_tcp_socket_recvfull failed|ERROR: Receiving from tomcat failed)/ ) {
+ $Global{tomcat_full}++;
+ $Interval{tomcat_full}++;
+ } elsif( $line =~ /(ajp_process_callback - write failed|ERROR sending data to client. Connection aborted or network problems|Client connection aborted or network problems)/ ) {
+ $Global{client_gone}++;
+ $Interval{client_gone}++;
+ }
+ next;
+ } else {
+ # Handle a mod_jk request log entry
+ $line =~ s/^\[.*\] //;
+ $line =~ s/\"(GET|POST|OPTIONS|HEAD)[^\"]*\" //;
+ $line =~ s/[\?\;].*\"//;
+ $line =~ s/\"//g;
+ ($work, $host, $page, $status, $latency) = split /\s+/,$line;
+ $page =~ s/\/\//\//g;
+ $page =~ s/\.\//\//g;
+ if( length($work) <= 0 || length($host) <= 0 ||
+ length($page) <= 0 || $status !~ /^\d+$/ || $latency !~ /^\d+\.\d+$/ ) {
+ print "Unknown log entry: $origline\n" unless $origline =~ /\.c /;
+ next;
+ }
+
+ # Throw out abnormally long requests and log them as an error
+ if( $latency >= 1800 ) {
+ $Global{errors} .= "Error: $page has an HTTP status of $status and an ";
+ $Global{errors} .= "abnormally long request latency of $latency seconds\n";
+ next;
+ }
+
+ # Save the data by day for Global, Worker, and Host
+ push @{$Global{latency}},$latency;
+ $workers = $Global{workers};
+ if( !defined $$workers{$work} ) {
+ undef @{"$work"};
+ undef %{"$work"};
+ undef %{"$work-hosts"};
+ ${"$work"}{latency} = \@{"$work"};
+ ${"$work"}{hosts} = \%{"$work-hosts"};
+ ${"$work"}{interval} = "";
+ $$workers{$work} = \%{"$work"};
+ }
+ $worker = $$workers{$work};
+ push @{$$worker{latency}},$latency;
+
+ if( !defined $$worker{hosts}{$host} ) {
+ undef @{"$work-$host"};
+ undef %{"$work-$host"};
+ undef %{"$work-$host-pages"};
+ ${"$work-$host"}{latency} = \@{"$work-$host"};
+ ${"$work-$host"}{pages} = \%{"$work-$host-pages"};
+ ${"$work-$host"}{interval} = "";
+ $$worker{hosts}{$host} = \%{"$work-$host"};
+ }
+ $hoster = $$worker{hosts}{$host};
+ push @{$$hoster{latency}},$latency;
+
+ if( !defined $$hoster{pages}{$page} ) {
+ undef @{"$work-$host-$page"};
+ $$hoster{pages}{$page} = \@{"$work-$host-$page"};
+ }
+ push @{$$hoster{pages}{$page}},$latency;
+
+ # Save the data by 5 minute interval for Global, Worker, and Host
+ push @{$Interval{latency}},$latency;
+ $workers = $Interval{workers};
+ if( !defined $$workers{"$work"} ) {
+ undef @{"int-$work"};
+ undef %{"int-$work"};
+ undef %{"int-$work-hosts"};
+ ${"int-$work"}{latency} = \@{"int-$work"};
+ ${"int-$work"}{hosts} = \%{"int-$work-hosts"};
+ $$workers{$work} = \%{"int-$work"};
+ }
+ $worker = $$workers{$work};
+ push @{$$worker{latency}},$latency;
+
+ if( !defined $$worker{hosts}{$host} ) {
+ undef @{"int-$work-$host"};
+ undef %{"int-$work-$host"};
+ ${"int-$work-$host"}{latency} = \@{"int-$work-$host"};
+ $$worker{hosts}{$host} = \%{"int-$work-$host"};
+ }
+ $hoster = $$worker{hosts}{$host};
+ push @{$$hoster{latency}},$latency;
+ }
+ }
+ }
+ undef $fh;
+}
+
+# If the last log file ends before switch to the current day,
+# output the last days data
+if( $logtime < $curdate ) {
+ &IntervalStats(\%Global,\%Interval,$previnterval*300);
+ &DailyStats($processdate,\%Global);
+}
+
+exit;
+
+sub IntervalStats($$$) {
+ my $global = $_[0];
+ my $data = $_[1];
+ my $interval = $_[2];
+
+ ($count,$median,$mean,$stddev,$min,$max) = &CalcStats($$data{latency});
+ $$global{interval} .= "$interval $count $median $mean $stddev $min $max $$data{client_gone} $$data{tomcat_full}\n";
+
+ foreach $work ( keys %{$$data{workers}} ) {
+ $worker = $$data{workers}{$work};
+ $gworker = $$global{workers}{$work};
+ ($count,$median,$mean,$stddev,$min,$max) = &CalcStats($$worker{latency});
+ $$gworker{interval} .= "$interval $count $median $mean $stddev $min $max\n";
+ foreach $host ( keys %{$$worker{hosts}} ) {
+ $hoster = $$worker{hosts}{$host};
+ $ghoster = $$gworker{hosts}{$host};
+ ($count,$median,$mean,$stddev,$min,$max) = &CalcStats($$hoster{latency});
+ $$ghoster{interval} .= "$interval $count $median $mean $stddev $min $max\n";
+ }
+ }
+}
+
+sub DailyStats($$) {
+ my $date = $_[0];
+ my $data = $_[1];
+
+ &SaveStats($data,$date,"","global");
+ &SaveFile($$data{interval},$date,"","daily");
+ foreach $work ( keys %{$$data{workers}} ) {
+ $worker = $$data{workers}{$work};
+ &SaveStats($worker,$date,$work,"global");
+ &SaveFile($$worker{interval},$date,$work,"daily");
+ foreach $host ( keys %{$$worker{hosts}} ) {
+ $hoster = $$worker{hosts}{$host};
+ &SaveStats($hoster,$date,"$work/$host","global");
+ &SaveFile($$hoster{interval},$date,"$work/$host","daily");
+ $pagedata = "";
+ foreach $page ( sort keys %{$$hoster{pages}} ) {
+ $pager = $$hoster{pages}{$page};
+ ($count,$median,$mean,$stddev,$min,$max) = &CalcStats($pager);
+ $pagedata .= "$page $count $median $mean $stddev $min $max\n";
+ }
+ $pagedata .= $$data{errors};
+ &SaveFile($pagedata,$date,"$work/$host","request");
+ }
+ }
+}
+
+sub CalcStats($) {
+ my $data = $_[0];
+
+ $stats = Statistics::Descriptive::Full->new();
+ $stats->add_data(@{$data});
+ $median = $stats->median();
+ $mean = $stats->mean();
+ $stddev = $stats->standard_deviation();
+ $max = $stats->max();
+ $min = $stats->min();
+ $count = $stats->count();
+ return ($count,$median,$mean,$stddev,$min,$max);
+}
+
+sub SaveStats($$$$) {
+ my $data = $_[0];
+ my $date = $_[1];
+ my $dir = $_[2];
+ my $file = $_[3];
+
+ if( length($dir) > 0 ) {
+ $dir = "$archivedir/$dir";
+ } else {
+ $dir = $archivedir;
+ }
+ mkdir "$dir",0755;
+
+ $outfile = "$dir/${file}.data";
+
+ ($count,$median,$mean,$stddev,$min,$max) = &CalcStats($$data{latency});
+
+ open DATA, ">>$outfile" or die $!;
+ print DATA "$date $count $median $mean $stddev $min $max";
+ print DATA " $$data{client_gone} $$data{tomcat_full}" if defined $$data{tomcat_full};
+ print DATA "\n";
+ close DATA;
+}
+
+sub SaveFile($$$$) {
+ my $data = $_[0];
+ my $date = $_[1];
+ my $dir = $_[2];
+ my $file = $_[3];
+ my ($day, $mon, $year);
+
+ ($day, $mon, $year) = (localtime($date))[3..5];
+ $year += 1900;
+ $mon++;
+ $mon = "0$mon" if $mon < 10;
+ $day = "0$day" if $day < 10;
+ $file = "$year-$mon-$day-$file";
+
+ if( length($dir) > 0 ) {
+ $dir = "$archivedir/$dir";
+ } else {
+ $dir = $archivedir;
+ }
+ mkdir "$dir",0755;
+
+ $outfile = "$dir/${file}.data";
+
+ open DATA, ">>$outfile" or die $!;
+ print DATA $data;
+ close DATA;
+}