Project

General

Profile

Download (8.69 KB) Statistics
| Branch: | Revision:
1 95b003ff Origo
#!/usr/bin/perl
2
3
# All rights reserved and Copyright (c) 2020 Origo Systems ApS.
4
# This file is provided with no warranty, and is subject to the terms and conditions defined in the license file LICENSE.md.
5
# The license file is part of this source code package and its content is also available at:
6
# https://www.origo.io/info/stabiledocs/licensing/stabile-open-source-license
7
8
9
# https://valve001.irigo.com/stabile/piston/stats.cgi?1.timestamp=1277045903&1.blk.hda.errs=-1&1.if.vnet1.tx_drop=0&1.if.vnet1.tx_packets=582932&1.if.vnet1.tx_errs=0&1.domain.nrVirtCpu=1&1.if.vnet1.tx_bytes=83963279&1.uuid=5eff51c7-f398-4eb9-8495-567cbb2bad77&1.if.vnet1.rx_packets=2898355&1.blk.hda.wr_req=183467&1.blk.hda.rd_req=206233&1.domain.cpuTime=26164280000000&1.domain.state=1&1.blk.hda.wr_bytes=3615029248&1.if.vnet1.rx_drop=0&1.domain.maxMem=2097152&1.if.vnet1.rx_bytes=1029898520&1.domain.memory=2097152&1.if.vnet1.rx_errs=0&1.blk.hda.rd_bytes=4088299520
10
# https://valve001.irigo.com/stabile/piston/stats.cgi?5.blk.hda.errs=-1&5.if.vnet1.tx_drop=0&5.if.vnet1.tx_packets=582932&5.if.vnet1.tx_errs=0&5.domain.nrVirtCpu=1&5.if.vnet1.tx_bytes=83963279&5.uuid=5eff51c7-f398-4eb9-8495-567cbb2bad77&5.if.vnet1.rx_packets=2898355&5.blk.hda.wr_req=183467&5.blk.hda.rd_req=206233&5.domain.cpuTime=26164280000000&5.domain.state=1&5.blk.hda.wr_bytes=3615029248&5.if.vnet1.rx_drop=0&5.domain.maxMem=2097152&5.if.vnet1.rx_bytes=1029898520&5.domain.memory=2097152&5.if.vnet1.rx_errs=0&5.blk.hda.rd_bytes=4088299520
11
#
12
# rrdtool dump 0de6349b-e10b-4465-90e4-d5a314263649.rrd | grep -v NaN|less
13
#
14
15
# Modifications made to valve001.irigo.com:
16
#   apt-get install apt-get install rrdtool librrd4 librrds-perl
17
#   mkdir /var/cache/rrdtool/
18
#   chown www-data:www-data /var/cache/rrdtool/
19
#
20
21
use CGI::Carp qw(fatalsToBrowser);
22
#use CGI 3.47 ':standard';
23
use CGI ':standard';
24
use JSON;
25
use URI::Escape;
26
use Error qw(:try);
27
use Sys::Syslog qw( :DEFAULT setlogsock);
28
use Data::Dumper;
29
use IO::Socket::INET;
30
use Python::Serialise::Pickle;
31
32
# Clear up tainted environment
33
$ENV{PATH} = '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin';
34
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};
35
36
my $q = new CGI;
37
my $params = $q->Vars;
38
39
$statsnap = 10; # time in seconds between stats updates - this must be coordinated with values in movepiston
40
41
42
print header;
43
#
44
# Put posted information into a hashtable for easy retrieval...
45
#
46
$hash = ();
47
48
while (my($key,$value) = each(%$params)) {
49
    @keysplit = split('\.', $key);
50
	$domainnumber = shift(@keysplit);
51
	$type = shift(@keysplit);
52
	$dev = shift(@keysplit);
53
	$param = shift(@keysplit);
54
55
	if ($type eq "domain") {
56
		$hash->{$domainnumber}->{$type}->{$dev} = $value;
57
	} elsif ($type eq "blk" or $type eq "if")	{
58
		$hash->{$domainnumber}->{$type}->{$dev}->{$param} = $value;
59
	}
60
}
61
62
# print(Dumper($hash));
63
64
65
#
66
# Find number of domains...
67
#
68
$numberOfDomainsSubmitted = getNumberOfDomainsSubmitted($params);
69
70
#
71
# Loop through all domain statistics, putting it in the database...
72
#
73
for ($i=1; $i <= $numberOfDomainsSubmitted; $i++) {
74
	my $uuid = $params->{$i.".uuid"};
75
    my $timestamp = int(getParam($params, "$i.timestamp"));
76
	print("Loop $i [".$uuid."]...\n");
77
78
#
79
# Make sure RRD for the uuid exists...
80
#
81
	checkRRD($uuid, $timestamp);
82
83
#
84
# Update RRD buckets...
85
#
86
#	my $cpuTime = getParam($params, "$i.domain.cpuTime") / getParam($params, "$i.domain.nrVirtCpu");
87
#    $cpuTime = 1 if ($cpuTime > 1);
88 71b897d3 hq
	my $cpuTime = getParam($params, "$i.domain.cpuTime")+0;
89
	my $cpuLoad = getParam($params, "$i.domain.cpuLoad")+0;
90
	my $rd_bytes = getParam($params, "$i.blk.hd.rd_bytes")+0;
91
	my $wr_bytes = getParam($params, "$i.blk.hd.wr_bytes")+0;
92
	my $rd_kbytes_s = getParam($params, "$i.blk.hd.rd_kbytes_s")+0;
93
	my $wr_kbytes_s = getParam($params, "$i.blk.hd.wr_kbytes_s")+0;
94
	my $rx_bytes = getParam($params, "$i.if.vnet.rx_bytes")+0;
95
	my $tx_bytes = getParam($params, "$i.if.vnet.tx_bytes")+0;
96
	my $rx_kbytes_s = getParam($params, "$i.if.vnet.rx_kbytes_s")+0;
97
	my $tx_kbytes_s = getParam($params, "$i.if.vnet.tx_kbytes_s")+0;
98
	my $nrVirtCpu = getParam($params, "$i.domain.nrVirtCpu")+0;
99
	my $memory = getParam($params, "$i.domain.memory")+0;
100
	my $maxMem = getParam($params, "$i.domain.maxMem")+0;
101 95b003ff Origo
102
	my $rrd_data = "$timestamp".
103
		":".$cpuTime.
104
		":".$rd_bytes.
105
		":".$wr_bytes.
106
		":".$rx_bytes.
107
		":".$tx_bytes.
108
		":".$nrVirtCpu.
109
		":".$memory.
110
		":".$maxMem.
111
		"";
112
	print "\t$rrd_data\n";
113
114
	# Update highres rrd...
115
	my $rrd_file_highres = "/var/cache/rrdtool/".$uuid."_highres.rrd";
116
	$rrd_file_highres = $1 if ($rrd_file_highres =~ /(.+)/);
117
	$rrd_data = $1 if ($rrd_data =~ /(.+)/);
118
	print("\tUpdating $rrd_file_highres...\n");
119
	print `/usr/bin/rrdtool update $rrd_file_highres $rrd_data`;
120
121
	my $pickle_data = [
122
		["domains.$uuid.cpuLoad", [$timestamp, $cpuLoad]],
123
		["domains.$uuid.rx_kbytes_s", [$timestamp, $rx_kbytes_s]],
124
		["domains.$uuid.tx_kbytes_s", [$timestamp, $tx_kbytes_s]],
125
		["domains.$uuid.rd_kbytes_s", [$timestamp, $rd_kbytes_s]],
126
		["domains.$uuid.wr_kbytes_s", [$timestamp, $wr_kbytes_s]],
127
		["domains.$uuid.rd_bytes", [$timestamp, $rd_bytes]],
128
		["domains.$uuid.wr_bytes", [$timestamp, $wr_bytes]],
129
		["domains.$uuid.rx_bytes", [$timestamp, $rx_bytes]],
130
		["domains.$uuid.tx_bytes", [$timestamp, $tx_bytes]],
131
		["domains.$uuid.nrVirtCpu", [$timestamp, $nrVirtCpu]],
132
		["domains.$uuid.memory", [$timestamp, $memory]],
133
		["domains.$uuid.maxMem", [$timestamp, $maxMem]]
134
	];
135
	print pickle_it($pickle_data);
136
137
}
138
139
print "---\nGot it, thanks!\n";
140
141
0;
142
143
sub getNumberOfDomainsSubmitted {
144
	my($parms) = @_;
145
    my $numberofdomains = 0;
146
    foreach my $dom (keys %$parms) {
147
        my @splits =  split('\.', $dom);
148
        my $domnum = shift(@splits);
149
        $numberofdomains = $domnum if ($domnum > $numberofdomains);
150
    }
151
	return($numberofdomains);
152
}
153
154
sub getParam {
155
	my($parms, $key) = @_;
156
	my $value = $parms->{$key};
157
	return $value;
158
#	return $value || 0;
159
}
160
161
#
162
# checkRRD: Checks if an RRD for the specified uuid exists. If not, a new RRD is created.
163
#
164
# @param uuid The uuid for which to look for an RRD for
165
#
166
sub checkRRD {
167
	my($uuid, $data_timestamp) = @_;
168
	my @rrd_order_counter = (
169
        "domain-cpuTime",
170
        "hda-rd_bytes",
171
        "hda-wr_bytes",
172
        "vnet0-rx_bytes",
173
        "vnet0-tx_bytes"
174
    );
175
	my @rrd_order_gauge = (
176
        "domain-nrVirtCpu",
177
        "domain-memory",
178
        "domain-maxMem",
179
    );
180
	my $rrd_datastores = "";
181
	foreach $dsname (@rrd_order_counter) {
182
		$rrd_datastores .= " DS:$dsname:COUNTER:30:U:U";
183
	}
184
	foreach $dsname (@rrd_order_gauge) {
185
		$rrd_datastores .= " DS:$dsname:GAUGE:30:U:U";
186
	}
187
188
	my $rrd_file_highres = "/var/cache/rrdtool/".$uuid."_highres.rrd";
189
	if ((not -e $rrd_file_highres) and ($uuid)) {
190
		#
191
		# RRD file for specified uuid not found, so create a new one...
192
		#
193
194
        my @rows_time_span = (
195
                            10,   # archive for next 10 mins
196
                            30,   # next 30 ...
197
                            60,   # next 60
198
                            120,  # next 2hrs
199
                            720,  # 12 hrs
200
                            1440, # 24 hrs
201
                            2880, # 2 days
202
                            14 * 24 * 60, # 14 days
203
                            28 * 24 * 60, # 28 days
204
                            6 * 30 * 24 * 60, # 6 months 
205
                           );
206
        my $extra = 5;
207
        my $rows = 30;
208
        my $rows_with_buffer = $rows + $extra;
209
        my @archives = ();
210
211
        foreach my $total_span ( @rows_time_span ) {
212
            # time span for each slot.
213
            $steps = ($total_span * 60) / $rows; # when sampling rate is pr. second
214
            $steps = $steps / $statsnap;         # since we are sampling every 10s
215
            # 0.9 XFiles Factor 90% of measurements can be UNKNOWN
216
            push @archives, "RRA:AVERAGE:0.9:$steps:$rows_with_buffer";
217
        }
218
219
        my $exec = "/usr/bin/rrdtool create $rrd_file_highres --start $data_timestamp --step $statsnap $rrd_datastores " . join(" ", @archives);
220
        print("Creating new highres RRD for $uuid...\n$exec\n");
221
        print `$exec` . "\n";
222
	}
223
}
224
225
sub pickle_it {
226
	my $data = shift;
227
	# example: [ ["path.mytest", [1332444075,27893687]], ["path.mytest", [1332444076,938.435]], ];
228
	my($carbon_server) = '127.0.0.1';
229
	my($carbon_port) = 2004;
230
231
232
	my($message) = pack("N/a*", pickle_dumps($data));
233
234
	my($sock) = IO::Socket::INET->new (
235
	                PeerAddr => $carbon_server,
236
	                PeerPort => $carbon_port,
237
	                Proto => 'tcp'
238
	            );
239
	            return "Unable to connect to pickle socket: $!\n" unless ($sock->connected);
240
241
	$sock->send($message);
242
	$sock->shutdown(2);
243
	return '';
244
}
245
246
# Work around P::S::Pickle 0.01's extremely limiting interface.
247
sub pickle_dumps {
248
   open(my $fh, '>', \my $s) or die $!;
249
   my $pickle = bless({ _fh => $fh }, 'Python::Serialise::Pickle');
250
   $pickle->dump($_[0]);
251
   $pickle->close();
252
   return $s;
253
}