Origo OS

#!/usr/bin/perl

# All rights reserved and Copyright (c) 2020 Origo Systems ApS.

# This file is provided with no warranty, and is subject to the terms and conditions defined in the license file LICENSE.md.

# The license file is part of this source code package and its content is also available at:

# https://www.origo.io/info/stabiledocs/licensing/stabile-open-source-license

# https://valve001.irigo.com/stabile/piston/stats.cgi?1.timestamp=1277045903&1.blk.hda.errs=-1&1.if.vnet1.tx_drop=0&1.if.vnet1.tx_packets=582932&1.if.vnet1.tx_errs=0&1.domain.nrVirtCpu=1&1.if.vnet1.tx_bytes=83963279&1.uuid=5eff51c7-f398-4eb9-8495-567cbb2bad77&1.if.vnet1.rx_packets=2898355&1.blk.hda.wr_req=183467&1.blk.hda.rd_req=206233&1.domain.cpuTime=26164280000000&1.domain.state=1&1.blk.hda.wr_bytes=3615029248&1.if.vnet1.rx_drop=0&1.domain.maxMem=2097152&1.if.vnet1.rx_bytes=1029898520&1.domain.memory=2097152&1.if.vnet1.rx_errs=0&1.blk.hda.rd_bytes=4088299520

# https://valve001.irigo.com/stabile/piston/stats.cgi?5.blk.hda.errs=-1&5.if.vnet1.tx_drop=0&5.if.vnet1.tx_packets=582932&5.if.vnet1.tx_errs=0&5.domain.nrVirtCpu=1&5.if.vnet1.tx_bytes=83963279&5.uuid=5eff51c7-f398-4eb9-8495-567cbb2bad77&5.if.vnet1.rx_packets=2898355&5.blk.hda.wr_req=183467&5.blk.hda.rd_req=206233&5.domain.cpuTime=26164280000000&5.domain.state=1&5.blk.hda.wr_bytes=3615029248&5.if.vnet1.rx_drop=0&5.domain.maxMem=2097152&5.if.vnet1.rx_bytes=1029898520&5.domain.memory=2097152&5.if.vnet1.rx_errs=0&5.blk.hda.rd_bytes=4088299520

#

# rrdtool dump 0de6349b-e10b-4465-90e4-d5a314263649.rrd | grep -v NaN|less

#

# Modifications made to valve001.irigo.com:

#   apt-get install apt-get install rrdtool librrd4 librrds-perl

#   mkdir /var/cache/rrdtool/

#   chown www-data:www-data /var/cache/rrdtool/

#

use CGI::Carp qw(fatalsToBrowser);

#use CGI 3.47 ':standard';

use CGI ':standard';

use JSON;

use URI::Escape;

use Error qw(:try);

use Sys::Syslog qw( :DEFAULT setlogsock);

use Data::Dumper;

use IO::Socket::INET;

use Python::Serialise::Pickle;

# Clear up tainted environment

$ENV{PATH} = '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin';

delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};

my $q = new CGI;

my $params = $q->Vars;

$statsnap = 10; # time in seconds between stats updates - this must be coordinated with values in movepiston

print header;

#

# Put posted information into a hashtable for easy retrieval...

#

$hash = ();

while (my($key,$value) = each(%$params)) {

    @keysplit = split('\.', $key);

	$domainnumber = shift(@keysplit);

	$type = shift(@keysplit);

	$dev = shift(@keysplit);

	$param = shift(@keysplit);

	if ($type eq "domain") {

		$hash->{$domainnumber}->{$type}->{$dev} = $value;

	} elsif ($type eq "blk" or $type eq "if")	{

		$hash->{$domainnumber}->{$type}->{$dev}->{$param} = $value;

	}

}

# print(Dumper($hash));

#

# Find number of domains...

#

$numberOfDomainsSubmitted = getNumberOfDomainsSubmitted($params);

#

# Loop through all domain statistics, putting it in the database...

#

for ($i=1; $i <= $numberOfDomainsSubmitted; $i++) {

	my $uuid = $params->{$i.".uuid"};

    my $timestamp = int(getParam($params, "$i.timestamp"));

	print("Loop $i [".$uuid."]...\n");

#

# Make sure RRD for the uuid exists...

#

	checkRRD($uuid, $timestamp);

#

# Update RRD buckets...

#

#	my $cpuTime = getParam($params, "$i.domain.cpuTime") / getParam($params, "$i.domain.nrVirtCpu");

#    $cpuTime = 1 if ($cpuTime > 1);

	my $cpuLoad = getParam($params, "$i.domain.cpuLoad")+0;

	my $rd_bytes = getParam($params, "$i.blk.hd.rd_bytes")+0;

	my $wr_bytes = getParam($params, "$i.blk.hd.wr_bytes")+0;

	my $rd_kbytes_s = getParam($params, "$i.blk.hd.rd_kbytes_s")+0;

	my $wr_kbytes_s = getParam($params, "$i.blk.hd.wr_kbytes_s")+0;

	my $rx_bytes = getParam($params, "$i.if.vnet.rx_bytes")+0;

	my $tx_bytes = getParam($params, "$i.if.vnet.tx_bytes")+0;

	my $rx_kbytes_s = getParam($params, "$i.if.vnet.rx_kbytes_s")+0;

	my $tx_kbytes_s = getParam($params, "$i.if.vnet.tx_kbytes_s")+0;

	my $nrVirtCpu = getParam($params, "$i.domain.nrVirtCpu")+0;

	my $memory = getParam($params, "$i.domain.memory")+0;

	my $maxMem = getParam($params, "$i.domain.maxMem")+0;

	my $rrd_data = "$timestamp".

		":".$cpuTime.

		":".$rd_bytes.

		":".$wr_bytes.

		":".$rx_bytes.

		":".$tx_bytes.

		":".$nrVirtCpu.

		":".$memory.

		":".$maxMem.

		"";

	print "\t$rrd_data\n";

	# Update highres rrd...

	my $rrd_file_highres = "/var/cache/rrdtool/".$uuid."_highres.rrd";

	$rrd_file_highres = $1 if ($rrd_file_highres =~ /(.+)/);

	$rrd_data = $1 if ($rrd_data =~ /(.+)/);

	print("\tUpdating $rrd_file_highres...\n");

	print `/usr/bin/rrdtool update $rrd_file_highres $rrd_data`;

	my $pickle_data = [

		["domains.$uuid.cpuLoad", [$timestamp, $cpuLoad]],

		["domains.$uuid.rx_kbytes_s", [$timestamp, $rx_kbytes_s]],

		["domains.$uuid.tx_kbytes_s", [$timestamp, $tx_kbytes_s]],

		["domains.$uuid.rd_kbytes_s", [$timestamp, $rd_kbytes_s]],

		["domains.$uuid.wr_kbytes_s", [$timestamp, $wr_kbytes_s]],

		["domains.$uuid.rd_bytes", [$timestamp, $rd_bytes]],

		["domains.$uuid.wr_bytes", [$timestamp, $wr_bytes]],

		["domains.$uuid.rx_bytes", [$timestamp, $rx_bytes]],

		["domains.$uuid.tx_bytes", [$timestamp, $tx_bytes]],

		["domains.$uuid.nrVirtCpu", [$timestamp, $nrVirtCpu]],

		["domains.$uuid.memory", [$timestamp, $memory]],

		["domains.$uuid.maxMem", [$timestamp, $maxMem]]

	];

	print pickle_it($pickle_data);

}

print "---\nGot it, thanks!\n";

0;

sub getNumberOfDomainsSubmitted {

	my($parms) = @_;

    my $numberofdomains = 0;

    foreach my $dom (keys %$parms) {

        my @splits =  split('\.', $dom);

        my $domnum = shift(@splits);

        $numberofdomains = $domnum if ($domnum > $numberofdomains);

    }

	return($numberofdomains);

}

sub getParam {

	my($parms, $key) = @_;

	my $value = $parms->{$key};

	return $value;

#	return $value || 0;

}

#

# checkRRD: Checks if an RRD for the specified uuid exists. If not, a new RRD is created.

#

# @param uuid The uuid for which to look for an RRD for

#

sub checkRRD {

	my($uuid, $data_timestamp) = @_;

	my @rrd_order_counter = (

        "domain-cpuTime",

        "hda-rd_bytes",

        "hda-wr_bytes",

        "vnet0-rx_bytes",

        "vnet0-tx_bytes"

    );

	my @rrd_order_gauge = (

        "domain-nrVirtCpu",

        "domain-memory",

        "domain-maxMem",

    );

	my $rrd_datastores = "";

	foreach $dsname (@rrd_order_counter) {

		$rrd_datastores .= " DS:$dsname:COUNTER:30:U:U";

	}

	foreach $dsname (@rrd_order_gauge) {

		$rrd_datastores .= " DS:$dsname:GAUGE:30:U:U";

	}

	my $rrd_file_highres = "/var/cache/rrdtool/".$uuid."_highres.rrd";

	if ((not -e $rrd_file_highres) and ($uuid)) {

		#

		# RRD file for specified uuid not found, so create a new one...

		#

        my @rows_time_span = (

                            10,   # archive for next 10 mins

                            30,   # next 30 ...

                            60,   # next 60

                            120,  # next 2hrs

                            720,  # 12 hrs

                            1440, # 24 hrs

                            2880, # 2 days

                            14 * 24 * 60, # 14 days

                            28 * 24 * 60, # 28 days

                            6 * 30 * 24 * 60, # 6 months 

                           );

        my $extra = 5;

        my $rows = 30;

        my $rows_with_buffer = $rows + $extra;

        my @archives = ();

        foreach my $total_span ( @rows_time_span ) {

            # time span for each slot.

            $steps = ($total_span * 60) / $rows; # when sampling rate is pr. second

            $steps = $steps / $statsnap;         # since we are sampling every 10s

            # 0.9 XFiles Factor 90% of measurements can be UNKNOWN

            push @archives, "RRA:AVERAGE:0.9:$steps:$rows_with_buffer";

        }

        my $exec = "/usr/bin/rrdtool create $rrd_file_highres --start $data_timestamp --step $statsnap $rrd_datastores " . join(" ", @archives);

        print("Creating new highres RRD for $uuid...\n$exec\n");

        print `$exec` . "\n";

	}

}

sub pickle_it {

	my $data = shift;

	# example: [ ["path.mytest", [1332444075,27893687]], ["path.mytest", [1332444076,938.435]], ];

	my($carbon_server) = '127.0.0.1';

	my($carbon_port) = 2004;

	my($message) = pack("N/a*", pickle_dumps($data));

	my($sock) = IO::Socket::INET->new (

	                PeerAddr => $carbon_server,

	                PeerPort => $carbon_port,

	                Proto => 'tcp'

	            );

	            return "Unable to connect to pickle socket: $!\n" unless ($sock->connected);

	$sock->send($message);

	$sock->shutdown(2);

	return '';

}

# Work around P::S::Pickle 0.01's extremely limiting interface.

sub pickle_dumps {

   open(my $fh, '>', \my $s) or die $!;

   my $pickle = bless({ _fh => $fh }, 'Python::Serialise::Pickle');

   $pickle->dump($_[0]);

   $pickle->close();

   return $s;

}