<html><head><meta name="color-scheme" content="light dark"></head><body><pre style="word-wrap: break-word; white-space: pre-wrap;">#!/usr/bin/env perl

# ========================================================================
# bncLogstash.pl
# ========================================================================
#
# Purpose: Reads BNC's logfile and converts the parsed metrics to InfluxDB
#          line protocol.
#
# Author  : Erwin Wiesensarter, April-2016
# Revision: $Header: trunk/BNC/scripts/bncLogstash.pl 9627 2022-02-21 09:07:55Z wiese $
# ========================================================================

# Uses
use strict;
use warnings;

BEGIN {
    use FindBin qw($Bin);
    use lib "$Bin";
}

use FindBin qw($Bin);
use Getopt::Long;
use File::Basename;
use File::Spec::Functions;
use List::MoreUtils qw(any uniq);
use Log::Log4perl qw(:easy);
use Time::Piece 1.30;
use Data::Dumper;
use POSIX qw(uname);

# BKG Perl libs
use Bnc;
use Common;

# Logging
Log::Log4perl-&gt;easy_init(
    {
      layout =&gt; '%d [%c] %p: %m%n',    # '[%F{1}-%L-%M]: %m%n'
      level  =&gt; $DEBUG                 # $TRACE $INFO
    }
);

# Options
my $bncConf   = {};
my ($home)    = glob "~";
my $corrMount = "n/a";
my $now       = Time::Piece-&gt;new;
my ($prog)    = fileparse($0);
my $hostname  = (uname)[1];
my $influx_db = 'realtime';

# Args
my $help          = 0;
my $fromBeginning = 0;
my $bncConfFile   = "";
my $intv          = 5;          # each epoch (default)
my $pppScene      = 0;
my $output        = "influx";
my @tags          = ();

GetOptions(
            'help'       =&gt; \$help,
            'from-begin' =&gt; \$fromBeginning,
            'conf=s'     =&gt; \$bncConfFile,
            'intv=s'     =&gt; \$intv,
            'scene=s'    =&gt; \$pppScene,
            'output=s'   =&gt; \$output,
            'tags=s'     =&gt; \@tags,
);

HELP_MESSAGE() if $help;

if ( $output &amp;&amp; !( $output =~ /influx/i ) ) { LOGDIE "Undefined output \"$output\"\n", HELP_MESSAGE() }
$output = lc $output;
$bncConfFile || LOGDIE( "Please set option \"conf\".\n", HELP_MESSAGE() );
if ( $bncConfFile &amp;&amp; !-s $bncConfFile ) {
    my $bncConfFile1 = catfile( $home, $bncConfFile );
    if   ( -s $bncConfFile1 ) { $bncConfFile = $bncConfFile1 }
    else                      { LOGDIE "BNC config file \"$bncConfFile\" does not exist\n" }
}
my %tags = ();
foreach (@tags) {
    s/^\s*//;
    s/\s+$//;
    my ( $k, $v ) = split ( /\s*=\s*/, $_, 2 );
    $tags{$k} = $v;
}

INFO "&gt;&gt;&gt;&gt;&gt; Start $prog $bncConfFile ..";

# Should only run once
# TODO checks args for that
Common::isAlreadyRunning() &amp;&amp; LOGDIE "Job is already running! Exit\n";

# -----------------------------------------------------------------------------
# Parse BNC config file
# -----------------------------------------------------------------------------
$bncConf   = Bnc::parseConfig($bncConfFile);     #  cmbMethod dataSource ?
$corrMount = $bncConf-&gt;{'PPP'}-&gt;{'corrMount'};
my $bncLogFile = $bncConf-&gt;{'General'}-&gt;{'logFile'};
my $date       = getLogFileDate($now);
$bncLogFile .= "_$date";                         # -&gt; *_160425

-s "$bncLogFile" || LOGDIE("logfile \"$bncLogFile\" does not exist. Exit\n");
my ( $logFilName, $logFilDir ) = fileparse($bncLogFile);

my $mpts_def = $bncConf-&gt;{'General'}-&gt;{'mountPoints_parsed'};
my %mpLookup = map { $_-&gt;{'mp'}, $_-&gt;{'caster'} } @$mpts_def;

# -----------------------------------------------------------------------------
# Select and parse BNC logfile
# -----------------------------------------------------------------------------
my $data   = Bnc::parseLogfile( $bncLogFile, $intv, !$fromBeginning );
my @EPOCHS = @{ $data-&gt;{'EPOCHS'} };
my @sites  = map { $_-&gt;{'site'} } @EPOCHS;
@sites = uniq @sites;
my $nof_epochs = scalar @EPOCHS;

my @latencies = @{ $data-&gt;{'LATENCIES'} };
my @restarts  = @{ $data-&gt;{'RESTARTS'} };

# Some output
DEBUG(   "\n&gt;   output   : $output\n&gt;   BNC conf : $bncConfFile"
       . "\n&gt;   BNC lfile: $bncLogFile\n&gt;   sampling : $intv\n&gt;   PPP-Scene: $pppScene"
       . "\n&gt;   Sites    : @sites\n&gt;   #epochs  : $nof_epochs\n&gt;   tags     : @tags"
       . "\n&gt;   Latencies: "
       . scalar @latencies
       . "\n&gt;   Restarts : "
       . scalar @restarts );

if ( $nof_epochs &lt; 1 &amp;&amp; scalar @latencies &lt; 1 ) {
    INFO "No epochs or latencies found. Do nothing";
    exit 0;
}

# -----------------------------------------------------------------------------
# Reformat data
# -----------------------------------------------------------------------------
my @data_influxFormated = ();

# Epochs
my $rcd = {};
foreach (@EPOCHS) {
    next if ( $_-&gt;{'dN'} == 0 || $_-&gt;{'dU'} == 0 );

    if ( $_-&gt;{'dN'} eq '-nan' || $_-&gt;{'dU'} eq '-nan' ) {
        next;
    }

    # Horizontal displacement (computation not available yet in influxDB)
    # influxDB kann sowas "SELECT sum(value * value) FROM cpu" leider NOCH nicht!
    # NEU in version 1.6: math functions SQRT,POW, usw. !!
    my $d2d = $_-&gt;{'dN'}**2 + $_-&gt;{'dE'}**2;    # Quadrat, um spaeter leichter den RMS zu berechnen
    my $d3d = $d2d + $_-&gt;{'dU'}**2;

    $rcd = {
        measurement =&gt; 'ppp',
        tags        =&gt; {
            mp        =&gt; $_-&gt;{'site'},
            corrMount =&gt; $corrMount || "NO",                       # SPP, empty geht derzeit nicht -&gt; InfluxError
            caster    =&gt; $mpLookup{ $_-&gt;{'site'} } || 'Unknown',
            host      =&gt; $hostname,
            scene     =&gt; $pppScene,
            clientSW  =&gt; 'BNC'

              #pppvers   =&gt; $pppvers,
              #bncvers   =&gt; '2.12.0',
        },
        fields =&gt; {
                    dN  =&gt; $_-&gt;{'dN'} + 0.0,
                    dE  =&gt; $_-&gt;{'dE'} + 0.0,
                    dU  =&gt; $_-&gt;{'dU'} + 0.0,
                    d2d =&gt; sprintf ( "%.5f", $d2d ),
                    d3d =&gt; sprintf ( "%.5f", $d3d ),
                    TRP =&gt; $_-&gt;{'TRP'} + 0.0,
        },
        time =&gt; $_-&gt;{'time'}
    };
    push ( @data_influxFormated, $rcd );
}

# Latencies
my $lat_rcd = {};
foreach (@latencies) {
    $lat_rcd = {
                 measurement =&gt; 'stream_latency',
                 tags        =&gt; {
                           mp     =&gt; $_-&gt;{'mp'},
                           type   =&gt; $_-&gt;{'type'},
                           caster =&gt; $mpLookup{ $_-&gt;{'mp'} } || 'Unknown',
                           host   =&gt; $hostname,
                           scene  =&gt; $pppScene
                 },
                 fields =&gt; {
                             meanLat =&gt; $_-&gt;{'meanLat'},
                             epochs  =&gt; $_-&gt;{'epochs'},
                             gaps    =&gt; $_-&gt;{'gaps'} || 0
                 },
                 time =&gt; $_-&gt;{'time'}
    };
    push ( @data_influxFormated, $lat_rcd );
}

# BNC Starts
if ($pppScene) {
    my $restart_rcd = {};
    foreach (@restarts) {
        $restart_rcd = {
                         measurement =&gt; 'ppp',
                         tags        =&gt; {
                                   host  =&gt; $hostname,
                                   scene =&gt; $pppScene,
                                   event =&gt; 'BNC_Start'
                         },
                         fields =&gt; { text =&gt; "\"BNC " . $_-&gt;{'bncvers'} . " started\"" },
                         time   =&gt; $_-&gt;{'time'}
        };
        push ( @data_influxFormated, $restart_rcd );
    }
}

# -----------------------------------------------------------------------------
# Output
# -----------------------------------------------------------------------------
if ( scalar @data_influxFormated &lt; 1 ) {
    INFO "No metrics found";
    exit 0;
}

# add tags from arguments
foreach my $rc (@data_influxFormated) {
    foreach my $k ( keys %tags ) {
        $rc-&gt;{'tags'}-&gt;{$k} = $tags{$k};
    }
}

if ( $output =~ /influx/ ) {
    my ( $i, $err ) = ( 0, 0 );
    foreach (@data_influxFormated) {
        my $l = Common::toLineProtocol($_);
        if ($l) {
            print STDOUT "$l\n";
            $i++;
        }
        else { $err++ }
    }
    DEBUG "Lines written: $i";
    DEBUG "Errorneous lines: $err";
}
else {
    LOGDIE("output not supported: $output");
}

##########################################################################
# Select date of the logfile we want to read
sub getLogFileDate {
    my ($tp) = @_;
    if ( $tp-&gt;hour == 0 &amp;&amp; $tp-&gt;min &lt; 5 ) {    # read last day
        $tp -= 300;
    }
    return $tp-&gt;strftime('%y%m%d');
}

sub HELP_MESSAGE {
    print &lt;&lt;EOI_HILFE;
$prog - reads BNC's logfile and writes metrics in formats that can be easily imported
by other tools, e.g. InfluxDB.

USAGE:
  $prog [paramter]

OPTIONS:
  -c|--conf        BNC config file for this job. Settings like PPP logpath
                   will be parsed from that config file.
                   Options -l and -b are not allowed together.
  -f|--from-begin  If set, the the logfile will be read from the beginning. Default is that the logfile is tailed.
  -o|--output      output format. So far only the InfluxDB line protocol (default).
  -i|--intv        sampling interval in seconds that shall be used in plots;
                   default: each epoch; for a daily logfile &lt;30&gt; seconds
                   should be usefull.
  -t|--tags &lt;tags&gt; comma separeted list of tags in format key=value,key2=value.
  -s|--scene       PPP scene number or identifier.
  -h|--help        show help contents.

EXAMPLES:
   $prog --conf path/to/BNC.bnc -i 5 -s 24 -f 2&gt;bnc.err
   $prog --conf path/to/BNC.bnc -o influx -t proj=cost,name=Tim 2&gt;bnc.err

Copyright (c) 2016 BKG Frankfurt &lt;erwin.wiesensarter\@bkg.bund.de&gt;
EOI_HILFE
    exit;
}
</pre></body></html>