#!/usr/bin/perl
#
# This script read specifically formatted nginx log and generate statistics about upstreams
#
# For use add to nginx config folowing lines:
#
# log_format timing '[$time_local] $remote_addr $status $body_bytes_sent $request_time [$upstream_addr] $upstream_response_time $http_host "$request"';
# access_log  /var/log/nginx/timing.log timing;
#
# usage: cat /var/log/nginx/timing.log | ./nginx_stat.pl
#

use strict;
use Data::Dumper qw(Dumper);
use List::Util qw(sum);

# Constatnts
my $MIN_REQUESTS_TO_SHOW = 100;
my $HOSTNAME_BIN = '/bin/hostname';

# Variables
my %domain_stats;
my %domain_calculated_stats;
my %responses_stats;
my %responses_totals;
my $tolal_requests;

while (<stdin>) {
    chomp;
    my $line = $_;

    my $domain;
    my $log_time; my $remote_addr; my $status; my $body_bytes_sent; my $request_time; my $upstream; my $upstream_response_time; my $http_host; my $request;

    # [28/Sep/2016:03:45:11 -0700] 66.249.69.159 200 159 0.003 [192.168.1.72:80] 0.003 1101-bill-joseph-way.spw4u.propertyminder.com "GET /robots.txt HTTP/1.1"
    if ($line =~ /^\[(.+)\] (\d+\.\d+\.\d+\.\d+) (\d+) (\d+) (\d+\.\d+) \[(.+)\] (.+) (.+) \".+\"/) {

        ($log_time, $remote_addr, $status, $body_bytes_sent, $request_time, $upstream, $upstream_response_time, $http_host, $request) = ($1, $2, $3, $4, $5, $6, $7, $8);
        # cut $http_host to domain level three 
        if ( $http_host =~ /.+\.(.+\..+\..+)$/ ) {  $domain = $1; }
        elsif ( $http_host =~ /.+\.(spw4u.com)$/ ) {  $domain = $1; }
        else {$domain = $http_host;}

        #
        # Count responses        
        #
        $tolal_requests++;
        if ( !$responses_stats{$domain} ) {
            %{$responses_stats{$domain}}=(
                'Tot'=>0,
                'Err'=>0,
                '1xx'=>0,
                '2xx'=>0,
                '3xx'=>0,
                '4xx'=>0,
                '5xx'=>0,
            );
        }
        $responses_stats{$domain}{'Tot'}++;
        if ($status =~ /1../) { $responses_stats{$domain}{'1xx'}++; }
        if ($status =~ /2../) { $responses_stats{$domain}{'2xx'}++; }
        if ($status =~ /3../) { $responses_stats{$domain}{'3xx'}++; }
        if ($status =~ /4../) { $responses_stats{$domain}{'4xx'}++; $responses_stats{$domain}{'Err'}++ }
        if ($status =~ /5../) { $responses_stats{$domain}{'5xx'}++; $responses_stats{$domain}{'Err'}++ }
        #
        # Count Upstreams response times
        #
        next if $upstream == "-"; # next if we don't have upstream
        push ( @{$domain_stats{$domain}{$upstream}{'arr_upstream_response_time'}} ,$upstream_response_time);
    }    
}

#
# Calculate upstream mean response time
#
foreach my $domain (keys %domain_stats ){
    foreach my $upstream (keys %{$domain_stats{$domain}} ){
        $domain_calculated_stats{$domain}{$upstream}{'adv_upstream_response_time'} = &mean( @{$domain_stats{$domain}{$upstream}{'arr_upstream_response_time'}} );
    }
}
#
# Output responses statistics
#
my $hostname = `$HOSTNAME_BIN`; chomp $hostname; 
$hostname = $hostname." ( ".`$HOSTNAME_BIN -I`; chomp $hostname;
$hostname = $hostname."):";

print "Nginx statistics from $hostname\n";
print "\n";
printf "%-10s%-10s%-10s%-10s%-10s%-10s%-10s%s\n", "Total","Errors","5xx","4xx","3xx","2xx","1xx", "Domain"; 
print  "============================================================================================\n";

foreach my $domain (sort { $responses_stats{$b}{"Err"} <=> $responses_stats{$a}{"Err"} } keys %responses_stats ) {
    next if $responses_stats{$domain}{"Tot"} < $MIN_REQUESTS_TO_SHOW;
    printf "%-10d%-10d%-10d%-10d%-10d%-10d%-10d%s\n", $responses_stats{$domain}{"Tot"},$responses_stats{$domain}{"Err"},$responses_stats{$domain}{"5xx"},$responses_stats{$domain}{"4xx"},$responses_stats{$domain}{"3xx"},$responses_stats{$domain}{"2xx"},$responses_stats{$domain}{"1xx"}, $domain; 
    foreach ('Tot', 'Err','5xx','4xx',"3xx","2xx","1xx"){
        $responses_totals{$_}=$responses_totals{$_}+$responses_stats{$domain}{$_};
    }
}
print  "--------------------------------------------------------------------------------------------\n";
printf "%-10s%-10s%-10s%-10s%-10s%-10s%-10s%s\n", $responses_totals{"Tot"},$responses_totals{"Err"},$responses_totals{"5xx"},$responses_totals{"4xx"},$responses_totals{"3xx"},$responses_totals{"2xx"},$responses_totals{"1xx"}, 'Totals'; 
#
# Output upstream response time
#
print "\n";
printf "%-10s%-10s%-21s%s\n", "Upstream","Total",   "","";
printf "%-10s%-10s%-21s%s\n", "response","upstream","","";
printf "%-10s%-10s%-21s%s\n", "time",    "requests","Upstream","Domain";
print  "=====================================================================\n";

for my $keypair (
        sort { $domain_calculated_stats{$b->[0]}{$b->[1]}{adv_upstream_response_time} <=> $domain_calculated_stats{$a->[0]}{$a->[1]}{adv_upstream_response_time} }
        map { my $intKey=$_; map [$intKey, $_], keys %{$domain_calculated_stats{$intKey}} } keys %domain_calculated_stats
    ) {
    my $total_upstream_requests = scalar(@{$domain_stats{$keypair->[0]}{$keypair->[1]}{'arr_upstream_response_time'}});
    next if $total_upstream_requests < $MIN_REQUESTS_TO_SHOW;
    my $domain = $keypair->[0];
    my $upstream = $keypair->[1];
    #printf "%.2f\t\t%s\t\t%-17s\t%s\n",  $domain_calculated_stats{$keypair->[0]}{$keypair->[1]}{'adv_upstream_response_time'}, $total_upstream_requests, $keypair->[1], $keypair->[0];
    printf "%-10.2f%-10s%-21s%s\n",  $domain_calculated_stats{$keypair->[0]}{$keypair->[1]}{'adv_upstream_response_time'}, $total_upstream_requests, $keypair->[1], $keypair->[0];
}

#
# Subroutines
#
sub mean {
    return sum(@_)/@_;
}