source: ntrip/trunk/BNC/scripts/bncLogstash.pl@ 10503

Last change on this file since 10503 was 9627, checked in by wiese, 3 years ago

add bncLogstash script

  • Property svn:executable set to *
  • Property svn:keywords set to Header
File size: 9.1 KB
Line 
1#!/usr/bin/env perl
2
3# ========================================================================
4# bncLogstash.pl
5# ========================================================================
6#
7# Purpose: Reads BNC's logfile and converts the parsed metrics to InfluxDB
8# line protocol.
9#
10# Author : Erwin Wiesensarter, April-2016
11# Revision: $Header: trunk/BNC/scripts/bncLogstash.pl 9627 2022-02-21 09:07:55Z wiese $
12# ========================================================================
13
14# Uses
15use strict;
16use warnings;
17
18BEGIN {
19 use FindBin qw($Bin);
20 use lib "$Bin";
21}
22
23use FindBin qw($Bin);
24use Getopt::Long;
25use File::Basename;
26use File::Spec::Functions;
27use List::MoreUtils qw(any uniq);
28use Log::Log4perl qw(:easy);
29use Time::Piece 1.30;
30use Data::Dumper;
31use POSIX qw(uname);
32
33# BKG Perl libs
34use Bnc;
35use Common;
36
37# Logging
38Log::Log4perl->easy_init(
39 {
40 layout => '%d [%c] %p: %m%n', # '[%F{1}-%L-%M]: %m%n'
41 level => $DEBUG # $TRACE $INFO
42 }
43);
44
45# Options
46my $bncConf = {};
47my ($home) = glob "~";
48my $corrMount = "n/a";
49my $now = Time::Piece->new;
50my ($prog) = fileparse($0);
51my $hostname = (uname)[1];
52my $influx_db = 'realtime';
53
54# Args
55my $help = 0;
56my $fromBeginning = 0;
57my $bncConfFile = "";
58my $intv = 5; # each epoch (default)
59my $pppScene = 0;
60my $output = "influx";
61my @tags = ();
62
63GetOptions(
64 'help' => \$help,
65 'from-begin' => \$fromBeginning,
66 'conf=s' => \$bncConfFile,
67 'intv=s' => \$intv,
68 'scene=s' => \$pppScene,
69 'output=s' => \$output,
70 'tags=s' => \@tags,
71);
72
73HELP_MESSAGE() if $help;
74
75if ( $output && !( $output =~ /influx/i ) ) { LOGDIE "Undefined output \"$output\"\n", HELP_MESSAGE() }
76$output = lc $output;
77$bncConfFile || LOGDIE( "Please set option \"conf\".\n", HELP_MESSAGE() );
78if ( $bncConfFile && !-s $bncConfFile ) {
79 my $bncConfFile1 = catfile( $home, $bncConfFile );
80 if ( -s $bncConfFile1 ) { $bncConfFile = $bncConfFile1 }
81 else { LOGDIE "BNC config file \"$bncConfFile\" does not exist\n" }
82}
83my %tags = ();
84foreach (@tags) {
85 s/^\s*//;
86 s/\s+$//;
87 my ( $k, $v ) = split ( /\s*=\s*/, $_, 2 );
88 $tags{$k} = $v;
89}
90
91INFO ">>>>> Start $prog $bncConfFile ..";
92
93# Should only run once
94# TODO checks args for that
95Common::isAlreadyRunning() && LOGDIE "Job is already running! Exit\n";
96
97# -----------------------------------------------------------------------------
98# Parse BNC config file
99# -----------------------------------------------------------------------------
100$bncConf = Bnc::parseConfig($bncConfFile); # cmbMethod dataSource ?
101$corrMount = $bncConf->{'PPP'}->{'corrMount'};
102my $bncLogFile = $bncConf->{'General'}->{'logFile'};
103my $date = getLogFileDate($now);
104$bncLogFile .= "_$date"; # -> *_160425
105
106-s "$bncLogFile" || LOGDIE("logfile \"$bncLogFile\" does not exist. Exit\n");
107my ( $logFilName, $logFilDir ) = fileparse($bncLogFile);
108
109my $mpts_def = $bncConf->{'General'}->{'mountPoints_parsed'};
110my %mpLookup = map { $_->{'mp'}, $_->{'caster'} } @$mpts_def;
111
112# -----------------------------------------------------------------------------
113# Select and parse BNC logfile
114# -----------------------------------------------------------------------------
115my $data = Bnc::parseLogfile( $bncLogFile, $intv, !$fromBeginning );
116my @EPOCHS = @{ $data->{'EPOCHS'} };
117my @sites = map { $_->{'site'} } @EPOCHS;
118@sites = uniq @sites;
119my $nof_epochs = scalar @EPOCHS;
120
121my @latencies = @{ $data->{'LATENCIES'} };
122my @restarts = @{ $data->{'RESTARTS'} };
123
124# Some output
125DEBUG( "\n> output : $output\n> BNC conf : $bncConfFile"
126 . "\n> BNC lfile: $bncLogFile\n> sampling : $intv\n> PPP-Scene: $pppScene"
127 . "\n> Sites : @sites\n> #epochs : $nof_epochs\n> tags : @tags"
128 . "\n> Latencies: "
129 . scalar @latencies
130 . "\n> Restarts : "
131 . scalar @restarts );
132
133if ( $nof_epochs < 1 && scalar @latencies < 1 ) {
134 INFO "No epochs or latencies found. Do nothing";
135 exit 0;
136}
137
138# -----------------------------------------------------------------------------
139# Reformat data
140# -----------------------------------------------------------------------------
141my @data_influxFormated = ();
142
143# Epochs
144my $rcd = {};
145foreach (@EPOCHS) {
146 next if ( $_->{'dN'} == 0 || $_->{'dU'} == 0 );
147
148 if ( $_->{'dN'} eq '-nan' || $_->{'dU'} eq '-nan' ) {
149 next;
150 }
151
152 # Horizontal displacement (computation not available yet in influxDB)
153 # influxDB kann sowas "SELECT sum(value * value) FROM cpu" leider NOCH nicht!
154 # NEU in version 1.6: math functions SQRT,POW, usw. !!
155 my $d2d = $_->{'dN'}**2 + $_->{'dE'}**2; # Quadrat, um spaeter leichter den RMS zu berechnen
156 my $d3d = $d2d + $_->{'dU'}**2;
157
158 $rcd = {
159 measurement => 'ppp',
160 tags => {
161 mp => $_->{'site'},
162 corrMount => $corrMount || "NO", # SPP, empty geht derzeit nicht -> InfluxError
163 caster => $mpLookup{ $_->{'site'} } || 'Unknown',
164 host => $hostname,
165 scene => $pppScene,
166 clientSW => 'BNC'
167
168 #pppvers => $pppvers,
169 #bncvers => '2.12.0',
170 },
171 fields => {
172 dN => $_->{'dN'} + 0.0,
173 dE => $_->{'dE'} + 0.0,
174 dU => $_->{'dU'} + 0.0,
175 d2d => sprintf ( "%.5f", $d2d ),
176 d3d => sprintf ( "%.5f", $d3d ),
177 TRP => $_->{'TRP'} + 0.0,
178 },
179 time => $_->{'time'}
180 };
181 push ( @data_influxFormated, $rcd );
182}
183
184# Latencies
185my $lat_rcd = {};
186foreach (@latencies) {
187 $lat_rcd = {
188 measurement => 'stream_latency',
189 tags => {
190 mp => $_->{'mp'},
191 type => $_->{'type'},
192 caster => $mpLookup{ $_->{'mp'} } || 'Unknown',
193 host => $hostname,
194 scene => $pppScene
195 },
196 fields => {
197 meanLat => $_->{'meanLat'},
198 epochs => $_->{'epochs'},
199 gaps => $_->{'gaps'} || 0
200 },
201 time => $_->{'time'}
202 };
203 push ( @data_influxFormated, $lat_rcd );
204}
205
206# BNC Starts
207if ($pppScene) {
208 my $restart_rcd = {};
209 foreach (@restarts) {
210 $restart_rcd = {
211 measurement => 'ppp',
212 tags => {
213 host => $hostname,
214 scene => $pppScene,
215 event => 'BNC_Start'
216 },
217 fields => { text => "\"BNC " . $_->{'bncvers'} . " started\"" },
218 time => $_->{'time'}
219 };
220 push ( @data_influxFormated, $restart_rcd );
221 }
222}
223
224# -----------------------------------------------------------------------------
225# Output
226# -----------------------------------------------------------------------------
227if ( scalar @data_influxFormated < 1 ) {
228 INFO "No metrics found";
229 exit 0;
230}
231
232# add tags from arguments
233foreach my $rc (@data_influxFormated) {
234 foreach my $k ( keys %tags ) {
235 $rc->{'tags'}->{$k} = $tags{$k};
236 }
237}
238
239if ( $output =~ /influx/ ) {
240 my ( $i, $err ) = ( 0, 0 );
241 foreach (@data_influxFormated) {
242 my $l = Common::toLineProtocol($_);
243 if ($l) {
244 print STDOUT "$l\n";
245 $i++;
246 }
247 else { $err++ }
248 }
249 DEBUG "Lines written: $i";
250 DEBUG "Errorneous lines: $err";
251}
252else {
253 LOGDIE("output not supported: $output");
254}
255
256##########################################################################
257# Select date of the logfile we want to read
258sub getLogFileDate {
259 my ($tp) = @_;
260 if ( $tp->hour == 0 && $tp->min < 5 ) { # read last day
261 $tp -= 300;
262 }
263 return $tp->strftime('%y%m%d');
264}
265
266sub HELP_MESSAGE {
267 print <<EOI_HILFE;
268$prog - reads BNC's logfile and writes metrics in formats that can be easily imported
269by other tools, e.g. InfluxDB.
270
271USAGE:
272 $prog [paramter]
273
274OPTIONS:
275 -c|--conf BNC config file for this job. Settings like PPP logpath
276 will be parsed from that config file.
277 Options -l and -b are not allowed together.
278 -f|--from-begin If set, the the logfile will be read from the beginning. Default is that the logfile is tailed.
279 -o|--output output format. So far only the InfluxDB line protocol (default).
280 -i|--intv sampling interval in seconds that shall be used in plots;
281 default: each epoch; for a daily logfile <30> seconds
282 should be usefull.
283 -t|--tags <tags> comma separeted list of tags in format key=value,key2=value.
284 -s|--scene PPP scene number or identifier.
285 -h|--help show help contents.
286
287EXAMPLES:
288 $prog --conf path/to/BNC.bnc -i 5 -s 24 -f 2>bnc.err
289 $prog --conf path/to/BNC.bnc -o influx -t proj=cost,name=Tim 2>bnc.err
290
291Copyright (c) 2016 BKG Frankfurt <erwin.wiesensarter\@bkg.bund.de>
292EOI_HILFE
293 exit;
294}
Note: See TracBrowser for help on using the repository browser.