#!/usr/bin/env perl
# ========================================================================
# bncLogstash.pl
# ========================================================================
#
# Purpose: Reads BNC's logfile and converts the parsed metrics to InfluxDB
# line protocol.
#
# Author : Erwin Wiesensarter, April-2016
# Revision: $Header: trunk/BNC/scripts/bncLogstash.pl 9627 2022-02-21 09:07:55Z wiese $
# ========================================================================
# Uses
use strict;
use warnings;
BEGIN {
use FindBin qw($Bin);
use lib "$Bin";
}
use FindBin qw($Bin);
use Getopt::Long;
use File::Basename;
use File::Spec::Functions;
use List::MoreUtils qw(any uniq);
use Log::Log4perl qw(:easy);
use Time::Piece 1.30;
use Data::Dumper;
use POSIX qw(uname);
# BKG Perl libs
use Bnc;
use Common;
# Logging
Log::Log4perl->easy_init(
{
layout => '%d [%c] %p: %m%n', # '[%F{1}-%L-%M]: %m%n'
level => $DEBUG # $TRACE $INFO
}
);
# Options
my $bncConf = {};
my ($home) = glob "~";
my $corrMount = "n/a";
my $now = Time::Piece->new;
my ($prog) = fileparse($0);
my $hostname = (uname)[1];
my $influx_db = 'realtime';
# Args
my $help = 0;
my $fromBeginning = 0;
my $bncConfFile = "";
my $intv = 5; # each epoch (default)
my $pppScene = 0;
my $output = "influx";
my @tags = ();
GetOptions(
'help' => \$help,
'from-begin' => \$fromBeginning,
'conf=s' => \$bncConfFile,
'intv=s' => \$intv,
'scene=s' => \$pppScene,
'output=s' => \$output,
'tags=s' => \@tags,
);
HELP_MESSAGE() if $help;
if ( $output && !( $output =~ /influx/i ) ) { LOGDIE "Undefined output \"$output\"\n", HELP_MESSAGE() }
$output = lc $output;
$bncConfFile || LOGDIE( "Please set option \"conf\".\n", HELP_MESSAGE() );
if ( $bncConfFile && !-s $bncConfFile ) {
my $bncConfFile1 = catfile( $home, $bncConfFile );
if ( -s $bncConfFile1 ) { $bncConfFile = $bncConfFile1 }
else { LOGDIE "BNC config file \"$bncConfFile\" does not exist\n" }
}
my %tags = ();
foreach (@tags) {
s/^\s*//;
s/\s+$//;
my ( $k, $v ) = split ( /\s*=\s*/, $_, 2 );
$tags{$k} = $v;
}
INFO ">>>>> Start $prog $bncConfFile ..";
# Should only run once
# TODO checks args for that
Common::isAlreadyRunning() && LOGDIE "Job is already running! Exit\n";
# -----------------------------------------------------------------------------
# Parse BNC config file
# -----------------------------------------------------------------------------
$bncConf = Bnc::parseConfig($bncConfFile); # cmbMethod dataSource ?
$corrMount = $bncConf->{'PPP'}->{'corrMount'};
my $bncLogFile = $bncConf->{'General'}->{'logFile'};
my $date = getLogFileDate($now);
$bncLogFile .= "_$date"; # -> *_160425
-s "$bncLogFile" || LOGDIE("logfile \"$bncLogFile\" does not exist. Exit\n");
my ( $logFilName, $logFilDir ) = fileparse($bncLogFile);
my $mpts_def = $bncConf->{'General'}->{'mountPoints_parsed'};
my %mpLookup = map { $_->{'mp'}, $_->{'caster'} } @$mpts_def;
# -----------------------------------------------------------------------------
# Select and parse BNC logfile
# -----------------------------------------------------------------------------
my $data = Bnc::parseLogfile( $bncLogFile, $intv, !$fromBeginning );
my @EPOCHS = @{ $data->{'EPOCHS'} };
my @sites = map { $_->{'site'} } @EPOCHS;
@sites = uniq @sites;
my $nof_epochs = scalar @EPOCHS;
my @latencies = @{ $data->{'LATENCIES'} };
my @restarts = @{ $data->{'RESTARTS'} };
# Some output
DEBUG( "\n> output : $output\n> BNC conf : $bncConfFile"
. "\n> BNC lfile: $bncLogFile\n> sampling : $intv\n> PPP-Scene: $pppScene"
. "\n> Sites : @sites\n> #epochs : $nof_epochs\n> tags : @tags"
. "\n> Latencies: "
. scalar @latencies
. "\n> Restarts : "
. scalar @restarts );
if ( $nof_epochs < 1 && scalar @latencies < 1 ) {
INFO "No epochs or latencies found. Do nothing";
exit 0;
}
# -----------------------------------------------------------------------------
# Reformat data
# -----------------------------------------------------------------------------
my @data_influxFormated = ();
# Epochs
my $rcd = {};
foreach (@EPOCHS) {
next if ( $_->{'dN'} == 0 || $_->{'dU'} == 0 );
if ( $_->{'dN'} eq '-nan' || $_->{'dU'} eq '-nan' ) {
next;
}
# Horizontal displacement (computation not available yet in influxDB)
# influxDB kann sowas "SELECT sum(value * value) FROM cpu" leider NOCH nicht!
# NEU in version 1.6: math functions SQRT,POW, usw. !!
my $d2d = $_->{'dN'}**2 + $_->{'dE'}**2; # Quadrat, um spaeter leichter den RMS zu berechnen
my $d3d = $d2d + $_->{'dU'}**2;
$rcd = {
measurement => 'ppp',
tags => {
mp => $_->{'site'},
corrMount => $corrMount || "NO", # SPP, empty geht derzeit nicht -> InfluxError
caster => $mpLookup{ $_->{'site'} } || 'Unknown',
host => $hostname,
scene => $pppScene,
clientSW => 'BNC'
#pppvers => $pppvers,
#bncvers => '2.12.0',
},
fields => {
dN => $_->{'dN'} + 0.0,
dE => $_->{'dE'} + 0.0,
dU => $_->{'dU'} + 0.0,
d2d => sprintf ( "%.5f", $d2d ),
d3d => sprintf ( "%.5f", $d3d ),
TRP => $_->{'TRP'} + 0.0,
},
time => $_->{'time'}
};
push ( @data_influxFormated, $rcd );
}
# Latencies
my $lat_rcd = {};
foreach (@latencies) {
$lat_rcd = {
measurement => 'stream_latency',
tags => {
mp => $_->{'mp'},
type => $_->{'type'},
caster => $mpLookup{ $_->{'mp'} } || 'Unknown',
host => $hostname,
scene => $pppScene
},
fields => {
meanLat => $_->{'meanLat'},
epochs => $_->{'epochs'},
gaps => $_->{'gaps'} || 0
},
time => $_->{'time'}
};
push ( @data_influxFormated, $lat_rcd );
}
# BNC Starts
if ($pppScene) {
my $restart_rcd = {};
foreach (@restarts) {
$restart_rcd = {
measurement => 'ppp',
tags => {
host => $hostname,
scene => $pppScene,
event => 'BNC_Start'
},
fields => { text => "\"BNC " . $_->{'bncvers'} . " started\"" },
time => $_->{'time'}
};
push ( @data_influxFormated, $restart_rcd );
}
}
# -----------------------------------------------------------------------------
# Output
# -----------------------------------------------------------------------------
if ( scalar @data_influxFormated < 1 ) {
INFO "No metrics found";
exit 0;
}
# add tags from arguments
foreach my $rc (@data_influxFormated) {
foreach my $k ( keys %tags ) {
$rc->{'tags'}->{$k} = $tags{$k};
}
}
if ( $output =~ /influx/ ) {
my ( $i, $err ) = ( 0, 0 );
foreach (@data_influxFormated) {
my $l = Common::toLineProtocol($_);
if ($l) {
print STDOUT "$l\n";
$i++;
}
else { $err++ }
}
DEBUG "Lines written: $i";
DEBUG "Errorneous lines: $err";
}
else {
LOGDIE("output not supported: $output");
}
##########################################################################
# Select date of the logfile we want to read
sub getLogFileDate {
my ($tp) = @_;
if ( $tp->hour == 0 && $tp->min < 5 ) { # read last day
$tp -= 300;
}
return $tp->strftime('%y%m%d');
}
sub HELP_MESSAGE {
print <<EOI_HILFE;
$prog - reads BNC's logfile and writes metrics in formats that can be easily imported
by other tools, e.g. InfluxDB.
USAGE:
$prog [paramter]
OPTIONS:
-c|--conf BNC config file for this job. Settings like PPP logpath
will be parsed from that config file.
Options -l and -b are not allowed together.
-f|--from-begin If set, the the logfile will be read from the beginning. Default is that the logfile is tailed.
-o|--output output format. So far only the InfluxDB line protocol (default).
-i|--intv sampling interval in seconds that shall be used in plots;
default: each epoch; for a daily logfile <30> seconds
should be usefull.
-t|--tags <tags> comma separeted list of tags in format key=value,key2=value.
-s|--scene PPP scene number or identifier.
-h|--help show help contents.
EXAMPLES:
$prog --conf path/to/BNC.bnc -i 5 -s 24 -f 2>bnc.err
$prog --conf path/to/BNC.bnc -o influx -t proj=cost,name=Tim 2>bnc.err
Copyright (c) 2016 BKG Frankfurt <erwin.wiesensarter\@bkg.bund.de>
EOI_HILFE
exit;
}