#!/bin/bash
# vim: expandtab:ts=4:sw=4:syntax=perl

# read rc files if exist
unset PROFILEDOTD
[ -e /etc/thruk/thruk.env  ] && . /etc/thruk/thruk.env
[ -e ~/etc/thruk/thruk.env ] && . ~/etc/thruk/thruk.env
[ -e ~/.thruk              ] && . ~/.thruk
[ -e ~/.profile            ] && . ~/.profile

BASEDIR=$(dirname $0)/..

# git version
if [ -d $BASEDIR/.git -a -e $BASEDIR/lib/Thruk.pm ]; then
  export PERL5LIB="$BASEDIR/lib:$PERL5LIB";
  if [ "$OMD_ROOT" != "" -a "$THRUK_CONFIG" = "" ]; then export THRUK_CONFIG="$OMD_ROOT/etc/thruk"; fi
  if [ "$THRUK_CONFIG" = "" ]; then export THRUK_CONFIG="$BASEDIR/"; fi

# omd
elif [ "$OMD_ROOT" != "" ]; then
  export PERL5LIB=$OMD_ROOT/share/thruk/lib:$PERL5LIB
  if [ "$THRUK_CONFIG" = "" ]; then export THRUK_CONFIG="$OMD_ROOT/etc/thruk"; fi

# pkg installation
else
  export PERL5LIB=$PERL5LIB:@DATADIR@/lib:@THRUKLIBS@;
  if [ "$THRUK_CONFIG" = "" ]; then export THRUK_CONFIG='@SYSCONFDIR@'; fi
fi

eval 'exec perl -x $0 ${1+"$@"} ;'
    if 0;
# / this slash makes vscode syntax highlighting work
#! -*- perl -*-
#line 35

##############################################
package WorkerCommandTester;

use strict;
use warnings;

use POSIX ();
use Pod::Usage;
use Getopt::Long;
use IO::Select ();
use IPC::Open3 qw/open3/;
use Thruk::Constants ':backend_handling';
use Thruk::Utils::CLI;
use Thruk::Utils::IO ();
use Thruk::Utils::Log qw/:all/;
use Thruk::Utils::Filter ();
use Thruk::Pool::Simple ();

# must come after the other modules
use Thread::Queue ();

$|=1; #enable autoflush
our $max_session_per_worker = 10;
our @sshpids;
my $sockets_queue = Thread::Queue->new();
our $exec;
our $terminal = -t STDOUT;

##############################################
sub END {
    while(my $pid = shift @sshpids) {
        CORE::kill(2, $pid)
    }
}

##############################################
sub new {
    my ($class, %arg) = @_;
    my $self = {
        continue_file => sprintf('/tmp/.worker.test.continue.%d.json', $>),
        summary       => {},
        stats         => {
            failed => 0,
            total  => 0,
        },
    };
    bless $self, $class;
    return $self;
}

##############################################
sub run {
    my($self) = @_;
    my $opt ={
        'help'          => 0,
        'hostsonly'     => undef,
        'hostfilter'    => undef,
        'servicefilter' => undef,
        'filter'        => undef,
        'backend'       => undef,
        'worker'        => 'auto',
        'exec'          => 'gearman',
        'target'        => [],
        'verbose'       => 0,
        'timeout'       => 30,
        'continue'      => undef,
        'retry'         => undef,
    };
    Getopt::Long::Configure('no_ignore_case');
    Getopt::Long::Configure('bundling');
    GetOptions (
    "h|help"          => \$opt->{'help'},
    "hostfilter=s"    => \$opt->{'hostfilter'},
    "hostsonly"       => \$opt->{'hostsonly'},
    "servicefilter=s" => \$opt->{'servicefilter'},
    "filter=s"        => \$opt->{'filter'},
    "b|backend=s"     => \$opt->{'backend'},
    "w|worker=i"      => \$opt->{'worker'},
    "e|exec=s"        => \$opt->{'exec'},
    "t|timeout=i"     => \$opt->{'timeout'},
    "c|continue"      => \$opt->{'continue'},
    "r|retry=s"       => \$opt->{'retry'},
    "v|verbose"       => sub { $opt->{'verbose'}++ },
    "<>"              => sub { push @{$opt->{'target'}}, $_[0]; },
    ) or pod2usage( { -verbose => 2, -message => 'error in options', -exit => 3 } );
    pod2usage( { -verbose => 2,  -exit => 3 } ) if($opt->{'help'} || scalar @{$opt->{'target'}} != 1);

    if($opt->{'retry'} && $opt->{'continue'}) {
        _error("retry file and continue option set, use either of them, not both.");
        exit(1);
    }
    if($opt->{'exec'} ne 'gearman' && $opt->{'exec'} ne 'shell') {
        _error("exec can be either 'gearman' or 'shell'");
        exit(1);
    }
    $exec = $opt->{'exec'};

    $ENV{'THRUK_VERBOSE'} = $opt->{'verbose'};
    Thruk::Config::set_config_env();

    my $options;
    $options->{'backends'} = [$opt->{'backend'}] if $opt->{'backend'};
    my $cli     = Thruk::Utils::CLI->new($options);
    my $c       = $cli->get_c();
    Thruk::Action::AddDefaults::add_defaults($c);
    $c->stash->{'backend_errors_handling'} = DIE;

    $self->{'c'}   = $c;
    $self->{'opt'} = $opt;

    if($opt->{'worker'} eq 'auto') {
        $opt->{'worker'} = 10;
    }

    for my $x (1..(POSIX::ceil($opt->{'worker'} / $max_session_per_worker ))) {
        my $pid = $self->_start_ssh_master($x-1);
        _debug("ssh control master %d started, pid:%d", $x-1, $pid);
        push @sshpids, $pid;
        my $ctrl_path = sprintf(".ssh-wrk-%s.%d", $opt->{'target'}->[0], $x-1);
        for(1..$max_session_per_worker) {
            $sockets_queue->enqueue($ctrl_path);
        }
    }

    # test ssh connection
    my($rc, $output, $err) = $self->_check_command({'line_expanded' => 'id' });
    if($rc != 0) {
        _error("ssh connection failed with rc %d", $rc);
        _error("%s", $output) if $output;
        _error("%s", $err) if $err;
        exit(1);
    }

    $rc = $self->_run_checks();
    $sockets_queue->end();
    if($rc) {
        return(0);
    }
    return(1);
}

##############################################
sub _run_checks {
    my($self) = @_;

    my $pool = Thruk::Utils::scale_out(
        scale        => $self->{'opt'}->{'worker'},
        worker       => sub { return($self->_worker(@_)); },
        prepare_only => 1,
    );

    $self->{'jobs_hash'} = {};
    my $jobs     = $self->_get_jobs();
    my $jobs_hash = $self->{'jobs_hash'};
    if(scalar keys %{$jobs_hash} == 0) {
        for my $job (@{$jobs}) {
            if($job->{name}) {
                $jobs_hash->{$job->{peer_key}}->{$job->{name}}->{'__HOST__'} = $job;
            } else {
                $jobs_hash->{$job->{peer_key}}->{$job->{host_name}}->{$job->{description}} = $job;
            }
        }
        $self->_save_continue_file();
    }

    $SIG{'INT'} = sub {
        _warn("got ctrl+c, printing summary so far before exiting...");
        $self->_end();
        exit(1);
    };

    my $t1       = time();
    my $num_jobs = scalar @{$jobs};
    my $numsize  = length("$num_jobs");
    my $nr       = 0;

    $pool->add_bulk($jobs);
    $pool->remove_all(sub {
        my($item) = @_;
        my $summary = $self->{'summary'};
        my($err, $obj) = @{$item};
        if($err) {
            $self->{'stats'}->{'failed'}++;
            if($obj->{'name'}) {
                $summary->{$obj->{'name'}}->{"__HOST__"} = {'address' => $obj->{'host_address'} // $obj->{'address'} };
            } else {
                $summary->{$obj->{'host_name'}}->{$obj->{'description'}} = {'address' => $obj->{'host_address'} // $obj->{'address'} };
            }
            _info("-" x 105);
            for my $l (@{$err}) {
                _info("%s", $l);
            }
        }
        $nr++;
        my $elapsed = time() - $t1;
        my($rem, $end, $perc, $rate) = ("", "", 0, 0);
        if($elapsed > 0) {
            $perc = ($nr/$num_jobs) * 100;
            my $rem_jobs = $num_jobs - $nr;
            $rate = $nr / $elapsed;
            if($rem_jobs > 0) {
                $rem = ($elapsed / $nr) * $rem_jobs;
                $end = time() + $rem;
                $rem = Thruk::Utils::Filter::duration($rem, 6);
                $end = Thruk::Utils::Filter::date_format($self->{'c'}, $end);
            }
        }
        printf("\033[JStatus: %0".$numsize."d/%d (%0.1f%%) check rate: %.1f/s | remaining duration: %s | expected end: %s \033[G",
            $nr,
            $num_jobs,
            $perc,
            $rate,
            $rem,
            $end,
        ) if $terminal;

        # clean up and save continue file information
        my $host_name;
        if($obj->{'name'}) {
            delete $jobs_hash->{$obj->{peer_key}}->{$obj->{name}}->{'__HOST__'};
            $host_name = $obj->{name};
        } else {
            delete $jobs_hash->{$obj->{peer_key}}->{$obj->{host_name}}->{$obj->{description}};
            $host_name = $obj->{host_name};
        }
        if(scalar keys %{$jobs_hash->{$obj->{peer_key}}->{$host_name}} == 0) {
            delete $jobs_hash->{$obj->{peer_key}}->{$host_name};
        }
        if($nr%5000 == 0) {
            $self->_save_continue_file();
        }

    });

    unlink($self->{'continue_file'});
    my $rc = $self->_print_summary();
    $self->_save_retry_file();
    $self->_end(1);
    return $rc;
}

##############################################
sub _end {
    my($self, $skip_summary) = @_;
    if($self && !$skip_summary) {
        $self->_print_summary();
        $self->_save_continue_file();
        _warn("you can continue the current run by starting $0 with --continue");
    }
    Thruk::Pool::Simple::shutdown();
}

##############################################
sub _save_continue_file {
    my($self) = @_;
    _debug("saving state file");
    Thruk::Utils::IO::json_store($self->{'continue_file'}, {
        jobs    => $self->{'jobs_hash'},
        summary => $self->{'summary'},
        stats   => $self->{'stats'},
    });
}

##############################################
sub _save_retry_file {
    my($self) = @_;
    _debug("saving retry file");

    return if scalar keys %{$self->{'summary'}} == 0;
    my $retry_file = $self->{'opt'}->{'retry'} || sprintf("/tmp/.worker.test.retry.%d.%s.json", $>, (POSIX::strftime("%Y%m%d_%H%M%S", localtime)));
    Thruk::Utils::IO::json_store($retry_file, {
        summary => $self->{'summary'},
    });
    _warn("retry failed checks with: %s ... --retry=%s", $0, $retry_file);
}

##############################################
sub _worker {
    my($self, $obj) = @_;

    local $SIG{'INT'} = sub { exit; };
    my $err;
    eval {
        ($err) = $self->_check_object($obj);
    };
    $err = [$@] if $@;
    return($err, $obj);
}

##############################################
sub _get_jobs {
    my($self) = @_;
    my $jobs = [];

    if($self->{'opt'}->{'continue'}) {
        if(! -e $self->{'continue_file'}) {
            _info("no continue file found, starting fresh");
        } else {
            my $data = Thruk::Utils::IO::json_lock_retrieve($self->{'continue_file'});
            $self->{'jobs_hash'} = $data->{'jobs'};
            $self->{'stats'}     = $data->{'stats'};
            $self->{'summary'}   = $data->{'summary'};
            my $jobs_hash = $self->{'jobs_hash'};
            for my $peer_key (sort keys %{$jobs_hash}) {
                for my $host_name (sort keys %{$jobs_hash->{$peer_key}}) {
                    push @{$jobs}, values %{$jobs_hash->{$peer_key}->{$host_name}};
                }
            }
            _warn("continuing remaining %d checks", scalar @{$jobs});
            return $jobs;
        }
    }

    my $retries;
    if($self->{'opt'}->{'retry'}) {
        if(! -e $self->{'opt'}->{'retry'}) {
            _error("error reading retry file %s: %s", $self->{'opt'}->{'retry'}, $!);
            exit(1);
        }
        my $data = Thruk::Utils::IO::json_lock_retrieve($self->{'opt'}->{'retry'});
        $retries = $data->{'summary'};
    }

    my $all_commands = {};
    my $c   = $self->{'c'};
    my $opt = $self->{'opt'};
    my $commands = $c->db->get_commands();
    _info("fetched %d commands", scalar @{$commands});
    for my $cmd (@{$commands}) {
        $all_commands->{$cmd->{'peer_key'}}->{$cmd->{'name'}} = $cmd;
    }

    my $last48hour    = time() - 2*86400;
    my $hostfilter    = [ { active_checks_enabled => 1, has_been_checked => 1, last_check => { '>=' => $last48hour } } ];
    my $servicefilter = [ { active_checks_enabled => 1, has_been_checked => 1, last_check => { '>=' => $last48hour } } ];
    if($opt->{'hostfilter'}) {
        push @{$hostfilter},    { name      => { '~~' => $opt->{'hostfilter'} }};
        push @{$servicefilter}, { host_name => { '~~' => $opt->{'hostfilter'} }};
    }
    if($opt->{'servicefilter'}) {
        push @{$servicefilter}, { description => { '~~' => $opt->{'servicefilter'} }};
    }
    my $has_hosts = 0;
    if($opt->{'filter'}) {
        my @names = split(/\s*;\s*/mx, $opt->{'filter'});
        my @filter;
        for my $name (@names) {
            if($name eq '__HOST__') {
                $has_hosts = 1;
                next;
            }
            push @filter, { description => $name };
        }
        if(scalar @filter > 0) {
            push @{$servicefilter}, Thruk::Utils::combine_filter( '-or', \@filter );
        } else {
            $opt->{'hostsonly'} = 1;
        }
    }
    if(!$opt->{'servicefilter'} || $has_hosts) {
        my $hosts = $c->db->get_hosts(filter => [$hostfilter], columns => [qw/name address check_command/]);
        push @{$jobs}, @{$hosts};
        _info("fetched %d hosts", scalar @{$hosts});
    }
    if(!$opt->{'hostsonly'}) {
        my $services = $c->db->get_services(filter => [$servicefilter], columns => [qw/host_name host_address description check_command/]);
        push @{$jobs}, @{$services};
        _info("fetched %d services", scalar @{$services});
    }

    # insert commands
    for my $obj (@{$jobs}) {
        my $check_command = $obj->{'check_command'};
        $check_command =~ s/\!.*$//gmx;
        my $cmd = $all_commands->{$obj->{'peer_key'}}->{$check_command};
        die("cannot find command definition for: ".$check_command) unless $cmd;
        $obj->{'command'} = $cmd;
    }

    if($retries) {
        my @filtered;
        for my $obj (@{$jobs}) {
            if($obj->{'name'}) {
                push @filtered, $obj if($retries->{$obj->{'name'}}->{"__HOST__"});
            } else {
                push @filtered, $obj if($retries->{$obj->{'host_name'}}->{$obj->{'description'}});
            }
        }
        $jobs = \@filtered;
        _info("retrying %d checks from %s", scalar @{$jobs}, $self->{'opt'}->{'retry'});
    }

    $self->{'stats'}->{'total'} = scalar @{$jobs};
    return($jobs);
}

##############################################
sub _print_summary {
    my($self) = @_;
    our $jobs;
    _info("=" x 105);
    _info("Checks failed on these hosts / services:");
    my $summary = $self->{'summary'};
    for my $hst (sort keys %{$summary}) {
        my @first = (sort keys %{$summary->{$hst}});
        _info("- %s   (address: %s)", $hst, $summary->{$hst}->{$first[0]}->{'address'});
        if($summary->{$hst}->{"__HOST__"}) {
            _info("  - host check");
        }
        for my $svc (sort keys %{$summary->{$hst}}) {
            next if $svc eq "__HOST__";
            _info("  - %s", $svc);
        }
    }

    _info("=" x 105);
    my $failed = $self->{'stats'}->{'failed'};
    my $total  = $self->{'stats'}->{'total'};
    if($failed == 0) {
        _info("%d/%d checks failed", $failed, $total);
        return(1);
    }
    _warn("%d/%d checks failed", $failed, $total);
    return;
}

##############################################
sub _check_object {
    my($self, $obj) = @_;

    my $c   = $self->{'c'};
    my $opt = $self->{'opt'};
    my($chkobj, $command, $name);
    # host check
    if($obj->{'name'}) {
        $name = $obj->{'name'};
        $chkobj = $c->db->get_hosts(filter => [{ name => $obj->{'name'} }], backend => [$obj->{'peer_key'}]);
        if(scalar @{$chkobj} != 1) {
            return([sprintf("found %d hosts for name %s", scalar @{$chkobj}, $name)]);
        }
        $chkobj = $chkobj->[0];
        $command = $c->db->expand_command('host' => $chkobj, 'command' => $obj->{'command'}, 'obfuscate' => 0 );
    } else {
        $name = sprintf("%s - %s", $obj->{'host_name'}, $obj->{'description'});
        $chkobj = $c->db->get_services(filter => [{ host_name => $obj->{'host_name'}, description => $obj->{'description'} }], backend => [$obj->{'peer_key'}]);
        if(scalar @{$chkobj} != 1) {
            return([sprintf("found %d services for name %s", scalar @{$chkobj}, $name)]);
        }
        $chkobj = $chkobj->[0];
        my $hosts = $c->db->get_hosts(filter => [{ name => $obj->{'host_name'} }], backend => [$obj->{'peer_key'}]);
        $command = $c->db->expand_command('host' => $hosts->[0], 'service' => $chkobj, 'command' => $obj->{'command'}, 'obfuscate' => 0 );
    }

    if($command->{'note'}) {
        return([
            sprintf("check command: %s", $command->{'line_expanded'}),
            sprintf("failed to expand check command (%s): %s",  $name, $command->{'note'}),
        ]);
    }
    if(!$command->{'line_expanded'}) {
        return([sprintf("got no command line for %s",  $name)]);
    }
    my($rc, $output, $err) = $self->_check_command($command);
    return $err if $err;
    $output = "" unless $output;
    $output =~ s/\n.*$//sgmx; # limit to first line
    $output =~ s/\\n.*$//sgmx; # limit to first line
    $output =~ s/^(.*?)\|.*$/$1/gmx; # strip perf data
    my $comp = $self->_compare_result($name, $chkobj, $rc, $output);
    if($comp == 0) {
        # everything fine
        return;
    }

    # changes detected
    return([
        sprintf("check command: %s", $command->{'line_expanded'}),
        sprintf("expected rc: %3s - output: %s (last_check: %s)", $chkobj->{'state'}, _shorten($chkobj->{'plugin_output'}, 80), Thruk::Utils::Filter::date_format($c, $chkobj->{'last_check'})),
        sprintf("actual   rc: %3s - output: %s", $rc, _shorten($output, 80)),
        sprintf("%s: %s has different %s",
            ($chkobj->{'name'} ? 'host' : 'service'),
            $name,
            ($comp == 1 ? 'exit code' : 'plugin output')),
    ]);
}

##############################################
# returns 0 if no changes were detected, 1 for exit code change and 2 for output changes
sub _compare_result {
    my($self, $name, $chkobj, $rc, $output) = @_;
    # translate host status
    if($chkobj->{'name'}) {
        if($chkobj->{'state'} == 0 && $rc != 0) {
            return 1;
        }
        if($chkobj->{'state'} != 0 && $rc == 0) {
            return 1;
        }
    } else {
        if($rc != $chkobj->{'state'}) {
            return 1;
        }
    }

    my $plugin_output = _make_plugin_output_comparable($chkobj->{'plugin_output'});
    my $tmp_output    = _make_plugin_output_comparable($output);
    if($plugin_output ne $tmp_output) {
        return 2;
    }

    return 0;
}

##############################################
sub _make_plugin_output_comparable {
    my($output) = @_;

    # trim whitespace
    $output =~ s/^\s+//gmx;
    $output =~ s/\s+$//gmx;

    # ping check output is a bit different sometimes
    $output =~ s/.*\Qrta nan, lost 100%\E.*/<HOST UNREACHABLE>/gmx;

    # make time out results more comparable
    $output =~ s/.*(timeout\ after\ \d+\ seconds).*/<TIMEOUT RESULT>/gmx;
    $output =~ s/.*(timed\ out\ after\ \d+\ seconds).*/<TIMEOUT RESULT>/gmx;
    $output =~ s/.*(Plugin\ timed\ out\ while\ executing\ system\ call).*/<TIMEOUT RESULT>/gmx;

    # replace dns errors, they might be slightly different
    $output =~ s/\QNo address associated with hostname\E/<DNS LOOKUP ERROR>/gmx; # rhel 9
    $output =~ s/\QName or service not known\E/<DNS LOOKUP ERROR>/gmx; # rhel 7

    $output =~ s/;/:/gmx; # naemon replaces semicolon with colons

    # replace numbers
    $output =~ s/\d+/<NUM>/gmx;


    return($output);
}

##############################################
sub _shorten {
    my($str, $len) = @_;
    if(length($str) > $len) {
        return(substr($str, 0, $len)."...");
    }
    return($str);
}

##############################################
sub _check_command {
    my($self, $command) = @_;

    my $ctrl_path = $sockets_queue->dequeue_timed(3); # get next free ctrl path
    die("got no ctrl path in time") unless $ctrl_path;
    my $target = $self->{'opt'}->{'target'}->[0];
    my $cmd = [];
    my $ssh_options = [
        "ssh",
        "-o", "PasswordAuthentication=no",
        "-o", "PreferredAuthentications=publickey",
        "-o", "IdentitiesOnly=yes",
        "-o", "ControlMaster=no",
        "-o", "ControlPath=".$ctrl_path,
        "-o", "StrictHostKeyChecking=no",
        $target,
    ];

    my $line_expanded = $command->{'line_expanded'};
    if($exec eq 'gearman') {
        $line_expanded =~ s/"/\\"/gmx;
        $line_expanded = '"'.$line_expanded.'"';
        $cmd = [
            @{$ssh_options},
            "./bin/mod_gearman_worker --job_timeout=".$self->{'opt'}->{'timeout'}." testcmd ".$line_expanded,
        ];
    }
    elsif($exec eq 'shell') {
        $cmd = [
            "timeout", "--kill-after=".($self->{'opt'}->{'timeout'}+2), $self->{'opt'}->{'timeout'},
            @{$ssh_options},
            $line_expanded,
        ];
    }

    my($rc, $output) = Thruk::Utils::IO::cmd($cmd, { no_touch_signals => 1 });
    if(($rc == 2 && $output =~ m/\Qtest run into timeout after\E/) || $rc == 124) {
        $rc     = 3;
        $output = sprintf("check timed out after %d seconds", $self->{'opt'}->{'timeout'});
    }
    $sockets_queue->enqueue($ctrl_path); # put it back
    return($rc, $output, undef);
}

##############################################
sub _start_ssh_master {
    my($self, $nr) = @_;
    my $target = $self->{'opt'}->{'target'}->[0];

    my $pid = fork();
    if($pid == -1) { die("fork failed"); }
    if($pid) { return($pid); }

    @sshpids = ();
    undef $Thruk::Globals::c;
    undef $self;
    my $ctrl_path = ".ssh-wrk-$target.".$nr;
    unlink($ctrl_path);
    my $cmd = [
        "-N",
        "-o", "PasswordAuthentication=no",
        "-o", "PreferredAuthentications=publickey",
        "-o", "IdentitiesOnly=yes",
        "-o", "ControlMaster=yes",
        "-o", "ControlPath=".$ctrl_path,
        "-o", "StrictHostKeyChecking=no",
        $target,
    ];
    my($wtr, $rdr, $err);
    _debug("starting: ssh ".join(" ", @{$cmd}));
    $pid = open3($wtr, $rdr, $err, "ssh", @{$cmd});
    $SIG{'INT'} = sub {
        CORE::kill(2, $pid) if $pid;
        exit(1);
    };
    my $sel = IO::Select->new;
    $sel->add($rdr);
    $sel->add($err);
    while(my @ready = $sel->can_read) {
        for my $fh (@ready) {
            my $line;
            my $len = sysread $fh, $line, 8192;
            if(!defined $len){
                die "Error from child: $!\n";
            } elsif ($len == 0){
                $sel->remove($fh);
                next;
            } else {
                my $prefix = (defined $rdr && $fh == $rdr) ? "OUT" : "ERR";
                for my $l (split/\n/mx, $line) {
                    _debug("SSHMASTER %s: %s", $prefix, $l);
                }
            }
        }
    }
    CORE::kill(2, $pid);
}

1;

##############################################
use strict;
use warnings;

##############################################

my $wrk = WorkerCommandTester->new();
exit($wrk->run());

##############################################

=head1 NAME

worker_command_tester - Test all command lines on a worker host

=head1 SYNOPSIS

Usage: worker_command_tester [options] user@targetmachine

=head1 DESCRIPTION

Runs all commands on given host and prints if they differ from the existing output.
Useful when testing a new mod-gearman worker for missing firewall
exceptions and similar.

=head1 OPTIONS

    -b|--backend            specify backend if there are multiple connected
       --hostsonly          only run hostchecks and skip services
       --hostfilter         filter hostnames by regular expression
       --servicefilter      filter service names by regular expression
       --filter             set a list of service names to check
    -w|--worker             specify the number of parallel checks
    -e|--exec               specify plugin execution, supported values: gearman, shell
    -t|--timeout            specify timeout (in seconds) for each check
    -c|--continue           try to continue previuous run
    -r|--retry=<file>       use retry file and verify failed runs
    -v|--verbose            print additional debug information

=head1 EXAMPLE

    # run checks from testhost only
    OMD[test]:~$ ./share/thruk/examples/worker_command_tester --hostfilter=testhost test@workerhost

    # run host checks only and write errors into a logfile
    OMD[test]:~$ ./share/thruk/examples/worker_command_tester --hostsonly test@workerhost 2> >(tee errors.log)

    # run only the host check and the ping service and write errors into a logfile
    OMD[test]:~$ ./share/thruk/examples/worker_command_tester --filter="__HOST__;Ping" test@workerhost 2> >(tee errors.log)

=head1 AUTHOR

2023, Sven Nierlein, <sven@nierlein.de>

=cut
