| 1 | #!/usr/bin/env perl
|
|---|
| 2 |
|
|---|
| 3 | # ========================================================================
|
|---|
| 4 | # bncLogstash.pl
|
|---|
| 5 | # ========================================================================
|
|---|
| 6 | #
|
|---|
| 7 | # Purpose: Reads BNC's logfile and converts the parsed metrics to InfluxDB
|
|---|
| 8 | # line protocol.
|
|---|
| 9 | #
|
|---|
| 10 | # Author : Erwin Wiesensarter, April-2016
|
|---|
| 11 | # Revision: $Header: trunk/BNC/scripts/bncLogstash.pl 9627 2022-02-21 09:07:55Z wiese $
|
|---|
| 12 | # ========================================================================
|
|---|
| 13 |
|
|---|
| 14 | # Uses
|
|---|
| 15 | use strict;
|
|---|
| 16 | use warnings;
|
|---|
| 17 |
|
|---|
| 18 | BEGIN {
|
|---|
| 19 | use FindBin qw($Bin);
|
|---|
| 20 | use lib "$Bin";
|
|---|
| 21 | }
|
|---|
| 22 |
|
|---|
| 23 | use FindBin qw($Bin);
|
|---|
| 24 | use Getopt::Long;
|
|---|
| 25 | use File::Basename;
|
|---|
| 26 | use File::Spec::Functions;
|
|---|
| 27 | use List::MoreUtils qw(any uniq);
|
|---|
| 28 | use Log::Log4perl qw(:easy);
|
|---|
| 29 | use Time::Piece 1.30;
|
|---|
| 30 | use Data::Dumper;
|
|---|
| 31 | use POSIX qw(uname);
|
|---|
| 32 |
|
|---|
| 33 | # BKG Perl libs
|
|---|
| 34 | use Bnc;
|
|---|
| 35 | use Common;
|
|---|
| 36 |
|
|---|
| 37 | # Logging
|
|---|
| 38 | Log::Log4perl->easy_init(
|
|---|
| 39 | {
|
|---|
| 40 | layout => '%d [%c] %p: %m%n', # '[%F{1}-%L-%M]: %m%n'
|
|---|
| 41 | level => $DEBUG # $TRACE $INFO
|
|---|
| 42 | }
|
|---|
| 43 | );
|
|---|
| 44 |
|
|---|
| 45 | # Options
|
|---|
| 46 | my $bncConf = {};
|
|---|
| 47 | my ($home) = glob "~";
|
|---|
| 48 | my $corrMount = "n/a";
|
|---|
| 49 | my $now = Time::Piece->new;
|
|---|
| 50 | my ($prog) = fileparse($0);
|
|---|
| 51 | my $hostname = (uname)[1];
|
|---|
| 52 | my $influx_db = 'realtime';
|
|---|
| 53 |
|
|---|
| 54 | # Args
|
|---|
| 55 | my $help = 0;
|
|---|
| 56 | my $fromBeginning = 0;
|
|---|
| 57 | my $bncConfFile = "";
|
|---|
| 58 | my $intv = 5; # each epoch (default)
|
|---|
| 59 | my $pppScene = 0;
|
|---|
| 60 | my $output = "influx";
|
|---|
| 61 | my @tags = ();
|
|---|
| 62 |
|
|---|
| 63 | GetOptions(
|
|---|
| 64 | 'help' => \$help,
|
|---|
| 65 | 'from-begin' => \$fromBeginning,
|
|---|
| 66 | 'conf=s' => \$bncConfFile,
|
|---|
| 67 | 'intv=s' => \$intv,
|
|---|
| 68 | 'scene=s' => \$pppScene,
|
|---|
| 69 | 'output=s' => \$output,
|
|---|
| 70 | 'tags=s' => \@tags,
|
|---|
| 71 | );
|
|---|
| 72 |
|
|---|
| 73 | HELP_MESSAGE() if $help;
|
|---|
| 74 |
|
|---|
| 75 | if ( $output && !( $output =~ /influx/i ) ) { LOGDIE "Undefined output \"$output\"\n", HELP_MESSAGE() }
|
|---|
| 76 | $output = lc $output;
|
|---|
| 77 | $bncConfFile || LOGDIE( "Please set option \"conf\".\n", HELP_MESSAGE() );
|
|---|
| 78 | if ( $bncConfFile && !-s $bncConfFile ) {
|
|---|
| 79 | my $bncConfFile1 = catfile( $home, $bncConfFile );
|
|---|
| 80 | if ( -s $bncConfFile1 ) { $bncConfFile = $bncConfFile1 }
|
|---|
| 81 | else { LOGDIE "BNC config file \"$bncConfFile\" does not exist\n" }
|
|---|
| 82 | }
|
|---|
| 83 | my %tags = ();
|
|---|
| 84 | foreach (@tags) {
|
|---|
| 85 | s/^\s*//;
|
|---|
| 86 | s/\s+$//;
|
|---|
| 87 | my ( $k, $v ) = split ( /\s*=\s*/, $_, 2 );
|
|---|
| 88 | $tags{$k} = $v;
|
|---|
| 89 | }
|
|---|
| 90 |
|
|---|
| 91 | INFO ">>>>> Start $prog $bncConfFile ..";
|
|---|
| 92 |
|
|---|
| 93 | # Should only run once
|
|---|
| 94 | # TODO checks args for that
|
|---|
| 95 | Common::isAlreadyRunning() && LOGDIE "Job is already running! Exit\n";
|
|---|
| 96 |
|
|---|
| 97 | # -----------------------------------------------------------------------------
|
|---|
| 98 | # Parse BNC config file
|
|---|
| 99 | # -----------------------------------------------------------------------------
|
|---|
| 100 | $bncConf = Bnc::parseConfig($bncConfFile); # cmbMethod dataSource ?
|
|---|
| 101 | $corrMount = $bncConf->{'PPP'}->{'corrMount'};
|
|---|
| 102 | my $bncLogFile = $bncConf->{'General'}->{'logFile'};
|
|---|
| 103 | my $date = getLogFileDate($now);
|
|---|
| 104 | $bncLogFile .= "_$date"; # -> *_160425
|
|---|
| 105 |
|
|---|
| 106 | -s "$bncLogFile" || LOGDIE("logfile \"$bncLogFile\" does not exist. Exit\n");
|
|---|
| 107 | my ( $logFilName, $logFilDir ) = fileparse($bncLogFile);
|
|---|
| 108 |
|
|---|
| 109 | my $mpts_def = $bncConf->{'General'}->{'mountPoints_parsed'};
|
|---|
| 110 | my %mpLookup = map { $_->{'mp'}, $_->{'caster'} } @$mpts_def;
|
|---|
| 111 |
|
|---|
| 112 | # -----------------------------------------------------------------------------
|
|---|
| 113 | # Select and parse BNC logfile
|
|---|
| 114 | # -----------------------------------------------------------------------------
|
|---|
| 115 | my $data = Bnc::parseLogfile( $bncLogFile, $intv, !$fromBeginning );
|
|---|
| 116 | my @EPOCHS = @{ $data->{'EPOCHS'} };
|
|---|
| 117 | my @sites = map { $_->{'site'} } @EPOCHS;
|
|---|
| 118 | @sites = uniq @sites;
|
|---|
| 119 | my $nof_epochs = scalar @EPOCHS;
|
|---|
| 120 |
|
|---|
| 121 | my @latencies = @{ $data->{'LATENCIES'} };
|
|---|
| 122 | my @restarts = @{ $data->{'RESTARTS'} };
|
|---|
| 123 |
|
|---|
| 124 | # Some output
|
|---|
| 125 | DEBUG( "\n> output : $output\n> BNC conf : $bncConfFile"
|
|---|
| 126 | . "\n> BNC lfile: $bncLogFile\n> sampling : $intv\n> PPP-Scene: $pppScene"
|
|---|
| 127 | . "\n> Sites : @sites\n> #epochs : $nof_epochs\n> tags : @tags"
|
|---|
| 128 | . "\n> Latencies: "
|
|---|
| 129 | . scalar @latencies
|
|---|
| 130 | . "\n> Restarts : "
|
|---|
| 131 | . scalar @restarts );
|
|---|
| 132 |
|
|---|
| 133 | if ( $nof_epochs < 1 && scalar @latencies < 1 ) {
|
|---|
| 134 | INFO "No epochs or latencies found. Do nothing";
|
|---|
| 135 | exit 0;
|
|---|
| 136 | }
|
|---|
| 137 |
|
|---|
| 138 | # -----------------------------------------------------------------------------
|
|---|
| 139 | # Reformat data
|
|---|
| 140 | # -----------------------------------------------------------------------------
|
|---|
| 141 | my @data_influxFormated = ();
|
|---|
| 142 |
|
|---|
| 143 | # Epochs
|
|---|
| 144 | my $rcd = {};
|
|---|
| 145 | foreach (@EPOCHS) {
|
|---|
| 146 | next if ( $_->{'dN'} == 0 || $_->{'dU'} == 0 );
|
|---|
| 147 |
|
|---|
| 148 | if ( $_->{'dN'} eq '-nan' || $_->{'dU'} eq '-nan' ) {
|
|---|
| 149 | next;
|
|---|
| 150 | }
|
|---|
| 151 |
|
|---|
| 152 | # Horizontal displacement (computation not available yet in influxDB)
|
|---|
| 153 | # influxDB kann sowas "SELECT sum(value * value) FROM cpu" leider NOCH nicht!
|
|---|
| 154 | # NEU in version 1.6: math functions SQRT,POW, usw. !!
|
|---|
| 155 | my $d2d = $_->{'dN'}**2 + $_->{'dE'}**2; # Quadrat, um spaeter leichter den RMS zu berechnen
|
|---|
| 156 | my $d3d = $d2d + $_->{'dU'}**2;
|
|---|
| 157 |
|
|---|
| 158 | $rcd = {
|
|---|
| 159 | measurement => 'ppp',
|
|---|
| 160 | tags => {
|
|---|
| 161 | mp => $_->{'site'},
|
|---|
| 162 | corrMount => $corrMount || "NO", # SPP, empty geht derzeit nicht -> InfluxError
|
|---|
| 163 | caster => $mpLookup{ $_->{'site'} } || 'Unknown',
|
|---|
| 164 | host => $hostname,
|
|---|
| 165 | scene => $pppScene,
|
|---|
| 166 | clientSW => 'BNC'
|
|---|
| 167 |
|
|---|
| 168 | #pppvers => $pppvers,
|
|---|
| 169 | #bncvers => '2.12.0',
|
|---|
| 170 | },
|
|---|
| 171 | fields => {
|
|---|
| 172 | dN => $_->{'dN'} + 0.0,
|
|---|
| 173 | dE => $_->{'dE'} + 0.0,
|
|---|
| 174 | dU => $_->{'dU'} + 0.0,
|
|---|
| 175 | d2d => sprintf ( "%.5f", $d2d ),
|
|---|
| 176 | d3d => sprintf ( "%.5f", $d3d ),
|
|---|
| 177 | TRP => $_->{'TRP'} + 0.0,
|
|---|
| 178 | },
|
|---|
| 179 | time => $_->{'time'}
|
|---|
| 180 | };
|
|---|
| 181 | push ( @data_influxFormated, $rcd );
|
|---|
| 182 | }
|
|---|
| 183 |
|
|---|
| 184 | # Latencies
|
|---|
| 185 | my $lat_rcd = {};
|
|---|
| 186 | foreach (@latencies) {
|
|---|
| 187 | $lat_rcd = {
|
|---|
| 188 | measurement => 'stream_latency',
|
|---|
| 189 | tags => {
|
|---|
| 190 | mp => $_->{'mp'},
|
|---|
| 191 | type => $_->{'type'},
|
|---|
| 192 | caster => $mpLookup{ $_->{'mp'} } || 'Unknown',
|
|---|
| 193 | host => $hostname,
|
|---|
| 194 | scene => $pppScene
|
|---|
| 195 | },
|
|---|
| 196 | fields => {
|
|---|
| 197 | meanLat => $_->{'meanLat'},
|
|---|
| 198 | epochs => $_->{'epochs'},
|
|---|
| 199 | gaps => $_->{'gaps'} || 0
|
|---|
| 200 | },
|
|---|
| 201 | time => $_->{'time'}
|
|---|
| 202 | };
|
|---|
| 203 | push ( @data_influxFormated, $lat_rcd );
|
|---|
| 204 | }
|
|---|
| 205 |
|
|---|
| 206 | # BNC Starts
|
|---|
| 207 | if ($pppScene) {
|
|---|
| 208 | my $restart_rcd = {};
|
|---|
| 209 | foreach (@restarts) {
|
|---|
| 210 | $restart_rcd = {
|
|---|
| 211 | measurement => 'ppp',
|
|---|
| 212 | tags => {
|
|---|
| 213 | host => $hostname,
|
|---|
| 214 | scene => $pppScene,
|
|---|
| 215 | event => 'BNC_Start'
|
|---|
| 216 | },
|
|---|
| 217 | fields => { text => "\"BNC " . $_->{'bncvers'} . " started\"" },
|
|---|
| 218 | time => $_->{'time'}
|
|---|
| 219 | };
|
|---|
| 220 | push ( @data_influxFormated, $restart_rcd );
|
|---|
| 221 | }
|
|---|
| 222 | }
|
|---|
| 223 |
|
|---|
| 224 | # -----------------------------------------------------------------------------
|
|---|
| 225 | # Output
|
|---|
| 226 | # -----------------------------------------------------------------------------
|
|---|
| 227 | if ( scalar @data_influxFormated < 1 ) {
|
|---|
| 228 | INFO "No metrics found";
|
|---|
| 229 | exit 0;
|
|---|
| 230 | }
|
|---|
| 231 |
|
|---|
| 232 | # add tags from arguments
|
|---|
| 233 | foreach my $rc (@data_influxFormated) {
|
|---|
| 234 | foreach my $k ( keys %tags ) {
|
|---|
| 235 | $rc->{'tags'}->{$k} = $tags{$k};
|
|---|
| 236 | }
|
|---|
| 237 | }
|
|---|
| 238 |
|
|---|
| 239 | if ( $output =~ /influx/ ) {
|
|---|
| 240 | my ( $i, $err ) = ( 0, 0 );
|
|---|
| 241 | foreach (@data_influxFormated) {
|
|---|
| 242 | my $l = Common::toLineProtocol($_);
|
|---|
| 243 | if ($l) {
|
|---|
| 244 | print STDOUT "$l\n";
|
|---|
| 245 | $i++;
|
|---|
| 246 | }
|
|---|
| 247 | else { $err++ }
|
|---|
| 248 | }
|
|---|
| 249 | DEBUG "Lines written: $i";
|
|---|
| 250 | DEBUG "Errorneous lines: $err";
|
|---|
| 251 | }
|
|---|
| 252 | else {
|
|---|
| 253 | LOGDIE("output not supported: $output");
|
|---|
| 254 | }
|
|---|
| 255 |
|
|---|
| 256 | ##########################################################################
|
|---|
| 257 | # Select date of the logfile we want to read
|
|---|
| 258 | sub getLogFileDate {
|
|---|
| 259 | my ($tp) = @_;
|
|---|
| 260 | if ( $tp->hour == 0 && $tp->min < 5 ) { # read last day
|
|---|
| 261 | $tp -= 300;
|
|---|
| 262 | }
|
|---|
| 263 | return $tp->strftime('%y%m%d');
|
|---|
| 264 | }
|
|---|
| 265 |
|
|---|
| 266 | sub HELP_MESSAGE {
|
|---|
| 267 | print <<EOI_HILFE;
|
|---|
| 268 | $prog - reads BNC's logfile and writes metrics in formats that can be easily imported
|
|---|
| 269 | by other tools, e.g. InfluxDB.
|
|---|
| 270 |
|
|---|
| 271 | USAGE:
|
|---|
| 272 | $prog [paramter]
|
|---|
| 273 |
|
|---|
| 274 | OPTIONS:
|
|---|
| 275 | -c|--conf BNC config file for this job. Settings like PPP logpath
|
|---|
| 276 | will be parsed from that config file.
|
|---|
| 277 | Options -l and -b are not allowed together.
|
|---|
| 278 | -f|--from-begin If set, the the logfile will be read from the beginning. Default is that the logfile is tailed.
|
|---|
| 279 | -o|--output output format. So far only the InfluxDB line protocol (default).
|
|---|
| 280 | -i|--intv sampling interval in seconds that shall be used in plots;
|
|---|
| 281 | default: each epoch; for a daily logfile <30> seconds
|
|---|
| 282 | should be usefull.
|
|---|
| 283 | -t|--tags <tags> comma separeted list of tags in format key=value,key2=value.
|
|---|
| 284 | -s|--scene PPP scene number or identifier.
|
|---|
| 285 | -h|--help show help contents.
|
|---|
| 286 |
|
|---|
| 287 | EXAMPLES:
|
|---|
| 288 | $prog --conf path/to/BNC.bnc -i 5 -s 24 -f 2>bnc.err
|
|---|
| 289 | $prog --conf path/to/BNC.bnc -o influx -t proj=cost,name=Tim 2>bnc.err
|
|---|
| 290 |
|
|---|
| 291 | Copyright (c) 2016 BKG Frankfurt <erwin.wiesensarter\@bkg.bund.de>
|
|---|
| 292 | EOI_HILFE
|
|---|
| 293 | exit;
|
|---|
| 294 | }
|
|---|
