pve-iscsi-zfs-scst/PVE/Storage/Custom/LunCmd/SCST.pm

package PVE::Storage::Custom::LunCmd::SCST;

# iscsi storage running SCST on Linux
#

#
#
# https://pve.proxmox.com/wiki/Storage:_ZFS_over_iSCSI
#
#
# mkdir /etc/pve/priv/zfs
# ssh-keygen -f /etc/pve/priv/zfs/<IP>_id_rsa
#
# ssh-copy-id -i /etc/pve/priv/zfs/<IP>_id_rsa.pub root@<IP>
#
# ssh -i /etc/pve/priv/zfs/<IP>_id_rsa root@<IP>
#
# On one of the proxmox nodes:
# 1) Login as root
# 2) ssh-copy-id <ip_of_iscsi_storage>
#
#
# On SCST LUN0 is ignored ...
# make sure it is always configured as a dummy-device
#
use strict;
use warnings;
use PVE::Tools qw(run_command file_read_firstline trim dir_glob_regex dir_glob_foreach);
use Data::Dumper;

use PVE::SafeSyslog;
use POSIX qw(strftime);
use File::Basename;

sub get_base;

# A logical unit can max have 16864 LUNs
# http://manpages.ubuntu.com/manpages/precise/man5/ietd.conf.5.html
#my $MAX_LUNS = 16864;
my $MAX_LUNS = 20; # more handy for development / dumping used luns

my $SETTINGS = undef;

my @ssh_opts = ('-o', 'BatchMode=yes');
my @ssh_cmd = ('/usr/bin/ssh', @ssh_opts);
my @scp_cmd = ('/usr/bin/scp', @ssh_opts);
my $id_rsa_path = '/etc/pve/priv/zfs';

my $scstadm = '/usr/local/sbin/scstadmin';


my $debugmsg = sub {
    my ($mtype, $msg, $logfd) = @_;

    chomp $msg;

    return if !$msg;

#    my $pre = $debugstattxt->{$mtype} || $debugstattxt->{'err'};
    my $pre = "SCST-ZFS";

    my $timestr = strftime ("%b %d %H:%M:%S", CORE::localtime);

    syslog ($mtype eq 'info' ? 'info' : 'err', "$pre $msg");

    foreach my $line (split (/\n/, $msg)) {
        print STDERR "$pre $line\n";
        print $logfd "$timestr $pre $line\n" if $logfd;
    }
};

my $execute_command = sub {
    my ($scfg, $exec, $timeout, $method, @params) = @_;

    my $msg = '';
    my $err = undef;
    my $target;
    my $cmd;
    my $res = ();

    $timeout = 10 if !$timeout;

    my $output = sub {
        my $line = shift;
        #$debugmsg->('info','#READ '.$line);
        $msg .= "$line\n";
    };

    my $errfunc = sub {
        my $line = shift;
        #$debugmsg->('info','#ERR '.$line);
        $err .= "$line";
    };

    if ($exec eq 'scp') {
        $target = 'root@[' . $scfg->{portal} . ']';
        $cmd = [@scp_cmd, '-i', "$id_rsa_path/$scfg->{portal}_id_rsa", '--', $method, "$target:$params[0]"];
    } else {
        $target = 'root@' . $scfg->{portal};
        $cmd = [@ssh_cmd, '-i', "$id_rsa_path/$scfg->{portal}_id_rsa", $target, '--', $method, @params];
    }

    eval {
        run_command($cmd, outfunc => $output, errfunc => $errfunc, timeout => $timeout);
    };
    if ($@) {
        $res = {
            result => 0,
            msg => $err,
        }
    } else {
        $res = {
            result => 1,
            msg => $msg,
        }
    }

    return $res;
};


my $get_target_path = sub {
    my ($scfg) = @_;
    if(defined $scfg->{ini_group} and length $scfg->{ini_group}) {
        return "/sys/kernel/scst_tgt/targets/iscsi/$scfg->{target}/ini_groups/$scfg->{ini_group}/luns/";
    } else {
        return "/sys/kernel/scst_tgt/targets/iscsi/$scfg->{target}/luns/";
    }
};


#
# Read config via SCST's sysfs, for the target in question
# no need for a parser with scst as we don't parse an actual config-file
#
# $SETTINS should list all LUN's
# what more ?
#
my $read_configured_luns = sub {
    my ($scfg, $timeout) = @_;
    my $msg = '';
    my $err = undef;
    my $target;
    my $targetPath;

    my $output = sub {
        my $line = shift;
        $msg .= "$line\n";
    };

    my $errfunc = sub {
        my $line = shift;
        $err .= "$line";
    };

    $timeout = 10 if !$timeout;


    $target = 'root@' . $scfg->{portal};

    $targetPath = $get_target_path->($scfg);


# esos find has no posix-extended and can't prinf !
#    my @listLunParams = (
#        $targetPath,
#        ,'-maxdepth','1'
#        ,'-regextype','posix-extended'
#        ,'-regex',"\'^.*/luns/\([0-9]+\)\$\'"
#        ,'-printf',"'%f\n'"
#    );


    my $findRegexp = "\'^.*luns/[0-9][0-9]\?[0-9]\?[0-9]\?\$'";
    if( defined $scfg->{esos} && $scfg->{esos} eq "1") {
        $findRegexp = "\'^.*luns/[0-9][0-9]\\?[0-9]\\?[0-9]\\?\$'";
    }
    my @listLunParams = (
        $targetPath,
        ,'-maxdepth','1'
        ,'-regex', $findRegexp
        ,'-print'
    );
    $debugmsg->('info',"find $targetPath -maxdepth 1 -regex $findRegexp -print");
    my $res = $execute_command->($scfg,'ssh',$timeout,'find',@listLunParams);
    die $res->{msg} unless $res->{result};

    my @lunsList = split "\n", $res->{msg};
    my $currentLun = undef;

    foreach (@lunsList) {
        $currentLun  = basename($_); # was genau kommt da ?
        if($currentLun != 0) {

            my $conf = undef;
            $conf->{include} = 1;
            $conf->{lun} = $currentLun;

            my $pathRes = $execute_command->($scfg, 'ssh', undef,
                'cat', "$targetPath$currentLun/device/filename");

            die $pathRes->{msg} unless $pathRes->{result};

            my @lunPath = split "\n", $pathRes->{msg};
            $conf->{Path} = $lunPath[0];

            push @{$SETTINGS->{luns}}, $conf;
        } else {
            # this is LUN 0 with vdisk_nullio
            my $conf = undef;
            $conf->{include} = 1;
            $conf->{lun} = $currentLun;
            $conf->{Path} = '/dev/null';

            push @{$SETTINGS->{luns}}, $conf;
        }
        $SETTINGS->{used}->{$currentLun} = 1;
    }
    return $res->{msg};
};


#
# read initial config, build settings and read luns
#
my $get_config = sub {
    my ($scfg) = @_;

    if(! defined $SETTINGS ) {
        $SETTINGS->{target} = $scfg->{target};
        $read_configured_luns->($scfg, undef);
    }
};

my $clear_config = sub {
    my ($scfg) = @_;
    $SETTINGS = undef;
};

#
# Write-out current config of running scst to /etc/scst.conf on Storage
# Makes use of use scstadmin --write_config
#
my $update_config = sub {
    my ($scfg) = @_;
    my $file = "/etc/scst.conf";

    my @params = ('--write_config',$file);
    my $res = $execute_command->($scfg, 'ssh',undef,'scstadmin', @params);
    die $res->{msg} unless $res->{result};

    if( defined $scfg->{esos} && $scfg->{esos} eq "1") {
        my $syncEsosRes = $execute_command->($scfg,'ssh',undef,'/usr/local/sbin/conf_sync.sh');
        die $syncEsosRes->{msg} unless $syncEsosRes->{result};
    }

};

my $get_target_tid = sub {
    my ($scfg) = @_;

    return 0;

    # what exactly is output of this ?
    # first * is iscsi
    # find /sys/kernel/scst_tgt/targets/*/iqn*/enabled 2> /dev/null
    # example-result for a enabled target:
    # /sys/kernel/scst_tgt/targets/iscsi/iqn.2016-06-02.modula-shop-systems.de:tgt/enabled

    my $proc = '/sys/kernel/scst_tgt/targets/iscsi/iqn*/enabled 2> /dev/null';
    my $tid = undef;

    my @params = ($proc);
    my $res = $execute_command->($scfg, 'ssh', undef, 'find', @params);
    die $res->{msg} unless $res->{result};
    my @cfg = split "\n", $res->{msg};

    foreach (@cfg) {

        #        $debugmsg->('info','target is : '.$scfg->{target}.' $_'. $_);
# gives:
# SCST-ZFS target is :
# iqn.2016-06-02.modula-shop-systems.de:tgt
# $_/sys/kernel/scst_tgt/targets/iscsi/iqn.2016-06-02.modula-shop-systems.de:tgt/enabled
        if ($_ =~ /^\s*tid:(\d+)\s+name:([\w\-\:\.]+)\s*$/) {
            if ($2 && $2 eq $scfg->{target}) {
                $tid = $1;
                last;
            }
        }
    }

    return $tid;
};

#
# Gets next free LUN-Number to be created
#
my $get_free_lun = sub {
    my $usedLuns = ();
    my $i=0;

    # LUN0 on SCST is reserved:
    $usedLuns->{$i} = 1;

    for ($i = 1; $i < $MAX_LUNS; $i++) {
        $usedLuns->{$i} = 0;
    }
    foreach my $lun (@{$SETTINGS->{luns}}) {
        $usedLuns->{$lun->{lun}} = 1;
    }
    $SETTINGS->{used} = $usedLuns;

    for ($i = 0; $i < $MAX_LUNS; $i++) {
        if(!$SETTINGS->{used}->{$i}) {
            #if(!$usedLuns->{$i}) {

#            $SETTINGS->{used}->{$i} = 1;
#            $debugmsg->('info',"get_lu_name result $i");
            return $i;
        }
    }
};

#
# Removes LUN from Settings and free`s the slot / LUN Number
#
my $free_lu_name = sub {
    my ($lu_name) = @_;
    my $updated_luns;

    foreach my $lun (@{$SETTINGS->{luns}}) {
        if ($lun->{lun} != $lu_name) {
            push @$updated_luns, $lun;
        }
    }
    $SETTINGS->{luns} = $updated_luns;
    $SETTINGS->{used}->{$lu_name} = undef;
};


#
# Creates a stub for a new LUN
#
my $make_lun = sub {
    my ($scfg, $path) = @_;

    die 'Maximum number of LUNs per target is $MAX_LUNS' if scalar @{$SETTINGS->{luns}} >= $MAX_LUNS;

    $get_config->($scfg);

    my $lun = $get_free_lun->();
    my $conf = {
        lun => $lun,
        Path => $path,
        Type => 'blockio',
        include => 1,
    };
    push @{$SETTINGS->{luns}}, $conf;

    return $conf;
};

my $list_view = sub {
    my ($scfg, $timeout, $method, @params) = @_;
    my $lun = undef;
    my $object = $params[0];

    foreach my $lun (@{$SETTINGS->{luns}}) {
        next unless $lun->{include} == 1;
        if ($lun->{Path} =~ /^$object$/) {
            return $lun->{lun} if (defined($lun->{lun}));
            die "$lun->{Path}: Missing LUN";
        }
    }

    return $lun;
};

#
# Returns Path of LUN when present, undef when Lun is not found
# Params: [0] - Path of LUN to search
#
my $list_lun = sub {
    my ($scfg, $timeout, $method, @params) = @_;
    my $name = undef;
    my $searchLun = $params[0];

    $clear_config->($scfg);
    $get_config->($scfg);

    foreach my $lun (@{$SETTINGS->{luns}}) {
        next unless $lun->{include} == 1;
        if ($lun->{Path} =~ /^$searchLun$/) {
            return $lun->{Path};
        }
    }
    return $name;
};

#
# Creates a new LUN on scst target
#
# First: crate device on scst
# Second: add the device to target with LUN
#
my $create_lun = sub {
    my ($scfg, $timeout, $method, @params) = @_;

    $clear_config->($scfg);
    $get_config->($scfg);

    if ($list_lun->($scfg, $timeout, $method, @params)) {
        die "$params[0]: LUN exists";
    }
    my $lun = $params[0];
    $lun = $make_lun->($scfg, $lun);
    my $tid = $get_target_tid->($scfg);
    my $path = "Path=$lun->{Path},Type=$lun->{Type}";

#    $debugmsg->('info', "CMD: --op new --tid=$tid --lun=$lun->{lun} --params $path");

    # add device in scst
    # echo "add_device disk1 filename=/disk1; blocksize=1" >/sys/kernel/scst_tgt/handlers/vdisk_fileio/mgmt
    #
    # add the device to target at lun:
    # echo "add disk1 0" >/sys/kernel/scst_tgt/targets/iscsi/iqn.2006-10.net.vlnb:tgt/luns/mgmt

    my $scstDiskName = basename($lun->{Path});

    my @paramsAddDevice = ('"','add_device',$scstDiskName,"filename=$lun->{Path}".'"',">/sys/kernel/scst_tgt/handlers/vdisk_blockio/mgmt");
    my $resAddDevice = $execute_command->($scfg, 'ssh', $timeout, 'echo', @paramsAddDevice);

    my $targetPath;
    $targetPath = $get_target_path->($scfg);
    my @paramsAddLun = (
        '"','add ',$scstDiskName,$lun->{lun},'"',
        ">$targetPath/mgmt"
    );
    my $resAddLun = $execute_command->($scfg, 'ssh', $timeout, 'echo', @paramsAddLun);
    do {
        my @paramsRemoveDevice = (
            "\"del_device $scstDiskName\"",
            ">/sys/kernel/scst_tgt/handlers/vdisk_blockio/mgmt"
        );

        my $resRemoveDevice = $execute_command->($scfg, 'ssh', $timeout, 'echo', @paramsRemoveDevice);

        $free_lu_name->($lun->{lun});
        $update_config->($scfg);
        die "Could not create LUN: $lun->{lun} PATH: $lun->{Path} NAME: $scstDiskName: ".$resAddLun->{msg};
    } unless $resAddLun->{result};

    # make config persist here ?
    $update_config->($scfg);
    $clear_config->($scfg);

    return $resAddLun->{msg};
};

my $delete_lun = sub {
    my ($scfg, $timeout, $method, @params) = @_;
    my $res = {msg => undef};
    $get_config->($scfg);

    my $path = $params[0];
    my $targetPath;
    $targetPath = $get_target_path->($scfg);
#    my $tid = $get_target_tid->($scfg);

#    $debugmsg->('info',"Delete LUN $path");
    foreach my $lun (@{$SETTINGS->{luns}}) {
        if ($lun->{Path} eq $path) {
#            $debugmsg->('info',"DO DELETE: $lun->{Path} $lun->{lun}");

            # Delete LUN
            @params = (
                "\"del $lun->{lun}\"",
                ">$targetPath/mgmt"
            );
            $res = $execute_command->($scfg, 'ssh', $timeout, 'echo', @params);
            if ($res->{result}) {
                $debugmsg->('info',"[OK - delete LUN]: $lun->{Path} : $res->{msg} -> call: free_lu_name: $lun->{lun}");

                # Delete Device
                my $scstDiskName = basename($lun->{Path});
                @params = (
                    "\"del_device $scstDiskName\"",
                    ">/sys/kernel/scst_tgt/handlers/vdisk_blockio/mgmt"
                );
                $res = $execute_command->($scfg, 'ssh', $timeout, 'echo', @params);

                if ($res->{result}) {
                    $free_lu_name->($lun->{lun});
                    $update_config->($scfg);
                    $clear_config->($scfg);
                    $get_config->($scfg);
                    last;
                } else {
                    die $res->{msg};
                }
            } else {
                die $res->{msg};
            }
        }
    }
    return $res->{msg};
};

my $import_lun = sub {
    my ($scfg, $timeout, $method, @params) = @_;

    return $create_lun->($scfg, $timeout, $method, @params);
};

my $modify_lun = sub {
    my ($scfg, $timeout, $method, @params) = @_;
    my $res;
    my $scstDevice = basename($params[1]);

    @params = ('echo 1',">/sys/kernel/scst_tgt/devices/$scstDevice/resync_size");
    $res = $execute_command->($scfg, 'ssh', $timeout, @params);
    die $res->{msg} unless $res->{result};

    return $res->{msg};
};

my $add_view = sub {
    my ($scfg, $timeout, $method, @params) = @_;

    return '';
};

my $get_lun_cmd_map = sub {
    my ($method) = @_;

    my $cmdmap = {
        create_lu   =>  { cmd => $create_lun },
        delete_lu   =>  { cmd => $delete_lun },
        import_lu   =>  { cmd => $import_lun },
        modify_lu   =>  { cmd => $modify_lun },
        add_view    =>  { cmd => $add_view },
        list_view   =>  { cmd => $list_view },
        list_lu     =>  { cmd => $list_lun },
    };

    die "unknown command '$method'" unless exists $cmdmap->{$method};

    return $cmdmap->{$method};
};

sub run_lun_command {
    my ($scfg, $timeout, $method, @params) = @_;
    $get_config->($scfg);
    my $cmdmap = $get_lun_cmd_map->($method);
    my $msg = $cmdmap->{cmd}->($scfg, $timeout, $method, @params);
    return $msg;
}

sub get_base {
    return '/dev';
}

1;