package PVE::Storage::Custom::LunCmd::SCST; # iscsi storage running SCST on Linux # # # # https://pve.proxmox.com/wiki/Storage:_ZFS_over_iSCSI # # # mkdir /etc/pve/priv/zfs # ssh-keygen -f /etc/pve/priv/zfs/_id_rsa # # ssh-copy-id -i /etc/pve/priv/zfs/_id_rsa.pub root@ # # ssh -i /etc/pve/priv/zfs/_id_rsa root@ # # On one of the proxmox nodes: # 1) Login as root # 2) ssh-copy-id # # # On SCST LUN0 is ignored ... # make sure it is always configured as a dummy-device # use strict; use warnings; use PVE::Tools qw(run_command file_read_firstline trim dir_glob_regex dir_glob_foreach); use Data::Dumper; use PVE::SafeSyslog; use POSIX qw(strftime); use File::Basename; sub get_base; # A logical unit can max have 16864 LUNs # http://manpages.ubuntu.com/manpages/precise/man5/ietd.conf.5.html #my $MAX_LUNS = 16864; my $MAX_LUNS = 20; # more handy for development / dumping used luns my $SETTINGS = undef; my @ssh_opts = ('-o', 'BatchMode=yes'); my @ssh_cmd = ('/usr/bin/ssh', @ssh_opts); my @scp_cmd = ('/usr/bin/scp', @ssh_opts); my $id_rsa_path = '/etc/pve/priv/zfs'; my $scstadm = '/usr/local/sbin/scstadmin'; my $debugmsg = sub { my ($mtype, $msg, $logfd) = @_; chomp $msg; return if !$msg; # my $pre = $debugstattxt->{$mtype} || $debugstattxt->{'err'}; my $pre = "SCST-ZFS"; my $timestr = strftime ("%b %d %H:%M:%S", CORE::localtime); syslog ($mtype eq 'info' ? 'info' : 'err', "$pre $msg"); foreach my $line (split (/\n/, $msg)) { print STDERR "$pre $line\n"; print $logfd "$timestr $pre $line\n" if $logfd; } }; my $execute_command = sub { my ($scfg, $exec, $timeout, $method, @params) = @_; my $msg = ''; my $err = undef; my $target; my $cmd; my $res = (); $timeout = 10 if !$timeout; my $output = sub { my $line = shift; #$debugmsg->('info','#READ '.$line); $msg .= "$line\n"; }; my $errfunc = sub { my $line = shift; #$debugmsg->('info','#ERR '.$line); $err .= "$line"; }; if ($exec eq 'scp') { $target = 'root@[' . $scfg->{portal} . ']'; $cmd = [@scp_cmd, '-i', "$id_rsa_path/$scfg->{portal}_id_rsa", '--', $method, "$target:$params[0]"]; } else { $target = 'root@' . $scfg->{portal}; $cmd = [@ssh_cmd, '-i', "$id_rsa_path/$scfg->{portal}_id_rsa", $target, '--', $method, @params]; } eval { run_command($cmd, outfunc => $output, errfunc => $errfunc, timeout => $timeout); }; if ($@) { $res = { result => 0, msg => $err, } } else { $res = { result => 1, msg => $msg, } } return $res; }; my $get_target_path = sub { my ($scfg) = @_; if(defined $scfg->{ini_group} and length $scfg->{ini_group}) { return "/sys/kernel/scst_tgt/targets/iscsi/$scfg->{target}/ini_groups/$scfg->{ini_group}/luns/"; } else { return "/sys/kernel/scst_tgt/targets/iscsi/$scfg->{target}/luns/"; } }; # # Read config via SCST's sysfs, for the target in question # no need for a parser with scst as we don't parse an actual config-file # # $SETTINS should list all LUN's # what more ? # my $read_configured_luns = sub { my ($scfg, $timeout) = @_; my $msg = ''; my $err = undef; my $target; my $targetPath; my $output = sub { my $line = shift; $msg .= "$line\n"; }; my $errfunc = sub { my $line = shift; $err .= "$line"; }; $timeout = 10 if !$timeout; $target = 'root@' . $scfg->{portal}; $targetPath = $get_target_path->($scfg); # esos find has no posix-extended and can't prinf ! # my @listLunParams = ( # $targetPath, # ,'-maxdepth','1' # ,'-regextype','posix-extended' # ,'-regex',"\'^.*/luns/\([0-9]+\)\$\'" # ,'-printf',"'%f\n'" # ); my $findRegexp = "\'^.*luns/[0-9][0-9]\?[0-9]\?[0-9]\?\$'"; if( defined $scfg->{esos} && $scfg->{esos} eq "1") { $findRegexp = "\'^.*luns/[0-9][0-9]\\?[0-9]\\?[0-9]\\?\$'"; } my @listLunParams = ( $targetPath, ,'-maxdepth','1' ,'-regex', $findRegexp ,'-print' ); $debugmsg->('info',"find $targetPath -maxdepth 1 -regex $findRegexp -print"); my $res = $execute_command->($scfg,'ssh',$timeout,'find',@listLunParams); die $res->{msg} unless $res->{result}; my @lunsList = split "\n", $res->{msg}; my $currentLun = undef; foreach (@lunsList) { $currentLun = basename($_); # was genau kommt da ? if($currentLun != 0) { my $conf = undef; $conf->{include} = 1; $conf->{lun} = $currentLun; my $pathRes = $execute_command->($scfg, 'ssh', undef, 'cat', "$targetPath$currentLun/device/filename"); die $pathRes->{msg} unless $pathRes->{result}; my @lunPath = split "\n", $pathRes->{msg}; $conf->{Path} = $lunPath[0]; push @{$SETTINGS->{luns}}, $conf; } else { # this is LUN 0 with vdisk_nullio my $conf = undef; $conf->{include} = 1; $conf->{lun} = $currentLun; $conf->{Path} = '/dev/null'; push @{$SETTINGS->{luns}}, $conf; } $SETTINGS->{used}->{$currentLun} = 1; } return $res->{msg}; }; # # read initial config, build settings and read luns # my $get_config = sub { my ($scfg) = @_; if(! defined $SETTINGS ) { $SETTINGS->{target} = $scfg->{target}; $read_configured_luns->($scfg, undef); } }; my $clear_config = sub { my ($scfg) = @_; $SETTINGS = undef; }; # # Write-out current config of running scst to /etc/scst.conf on Storage # Makes use of use scstadmin --write_config # my $update_config = sub { my ($scfg) = @_; my $file = "/etc/scst.conf"; my @params = ('--write_config',$file); my $res = $execute_command->($scfg, 'ssh',undef,'scstadmin', @params); die $res->{msg} unless $res->{result}; if( defined $scfg->{esos} && $scfg->{esos} eq "1") { my $syncEsosRes = $execute_command->($scfg,'ssh',undef,'/usr/local/sbin/conf_sync.sh'); die $syncEsosRes->{msg} unless $syncEsosRes->{result}; } }; my $get_target_tid = sub { my ($scfg) = @_; return 0; # what exactly is output of this ? # first * is iscsi # find /sys/kernel/scst_tgt/targets/*/iqn*/enabled 2> /dev/null # example-result for a enabled target: # /sys/kernel/scst_tgt/targets/iscsi/iqn.2016-06-02.modula-shop-systems.de:tgt/enabled my $proc = '/sys/kernel/scst_tgt/targets/iscsi/iqn*/enabled 2> /dev/null'; my $tid = undef; my @params = ($proc); my $res = $execute_command->($scfg, 'ssh', undef, 'find', @params); die $res->{msg} unless $res->{result}; my @cfg = split "\n", $res->{msg}; foreach (@cfg) { # $debugmsg->('info','target is : '.$scfg->{target}.' $_'. $_); # gives: # SCST-ZFS target is : # iqn.2016-06-02.modula-shop-systems.de:tgt # $_/sys/kernel/scst_tgt/targets/iscsi/iqn.2016-06-02.modula-shop-systems.de:tgt/enabled if ($_ =~ /^\s*tid:(\d+)\s+name:([\w\-\:\.]+)\s*$/) { if ($2 && $2 eq $scfg->{target}) { $tid = $1; last; } } } return $tid; }; # # Gets next free LUN-Number to be created # my $get_free_lun = sub { my $usedLuns = (); my $i=0; # LUN0 on SCST is reserved: $usedLuns->{$i} = 1; for ($i = 1; $i < $MAX_LUNS; $i++) { $usedLuns->{$i} = 0; } foreach my $lun (@{$SETTINGS->{luns}}) { $usedLuns->{$lun->{lun}} = 1; } $SETTINGS->{used} = $usedLuns; for ($i = 0; $i < $MAX_LUNS; $i++) { if(!$SETTINGS->{used}->{$i}) { #if(!$usedLuns->{$i}) { # $SETTINGS->{used}->{$i} = 1; # $debugmsg->('info',"get_lu_name result $i"); return $i; } } }; # # Removes LUN from Settings and free`s the slot / LUN Number # my $free_lu_name = sub { my ($lu_name) = @_; my $updated_luns; foreach my $lun (@{$SETTINGS->{luns}}) { if ($lun->{lun} != $lu_name) { push @$updated_luns, $lun; } } $SETTINGS->{luns} = $updated_luns; $SETTINGS->{used}->{$lu_name} = undef; }; # # Creates a stub for a new LUN # my $make_lun = sub { my ($scfg, $path) = @_; die 'Maximum number of LUNs per target is $MAX_LUNS' if scalar @{$SETTINGS->{luns}} >= $MAX_LUNS; $get_config->($scfg); my $lun = $get_free_lun->(); my $conf = { lun => $lun, Path => $path, Type => 'blockio', include => 1, }; push @{$SETTINGS->{luns}}, $conf; return $conf; }; my $list_view = sub { my ($scfg, $timeout, $method, @params) = @_; my $lun = undef; my $object = $params[0]; foreach my $lun (@{$SETTINGS->{luns}}) { next unless $lun->{include} == 1; if ($lun->{Path} =~ /^$object$/) { return $lun->{lun} if (defined($lun->{lun})); die "$lun->{Path}: Missing LUN"; } } return $lun; }; # # Returns Path of LUN when present, undef when Lun is not found # Params: [0] - Path of LUN to search # my $list_lun = sub { my ($scfg, $timeout, $method, @params) = @_; my $name = undef; my $searchLun = $params[0]; $clear_config->($scfg); $get_config->($scfg); foreach my $lun (@{$SETTINGS->{luns}}) { next unless $lun->{include} == 1; if ($lun->{Path} =~ /^$searchLun$/) { return $lun->{Path}; } } return $name; }; # # Creates a new LUN on scst target # # First: crate device on scst # Second: add the device to target with LUN # my $create_lun = sub { my ($scfg, $timeout, $method, @params) = @_; $clear_config->($scfg); $get_config->($scfg); if ($list_lun->($scfg, $timeout, $method, @params)) { die "$params[0]: LUN exists"; } my $lun = $params[0]; $lun = $make_lun->($scfg, $lun); my $tid = $get_target_tid->($scfg); my $path = "Path=$lun->{Path},Type=$lun->{Type}"; # $debugmsg->('info', "CMD: --op new --tid=$tid --lun=$lun->{lun} --params $path"); # add device in scst # echo "add_device disk1 filename=/disk1; blocksize=1" >/sys/kernel/scst_tgt/handlers/vdisk_fileio/mgmt # # add the device to target at lun: # echo "add disk1 0" >/sys/kernel/scst_tgt/targets/iscsi/iqn.2006-10.net.vlnb:tgt/luns/mgmt my $scstDiskName = basename($lun->{Path}); my @paramsAddDevice = ('"','add_device',$scstDiskName,"filename=$lun->{Path}".'"',">/sys/kernel/scst_tgt/handlers/vdisk_blockio/mgmt"); my $resAddDevice = $execute_command->($scfg, 'ssh', $timeout, 'echo', @paramsAddDevice); my $targetPath; $targetPath = $get_target_path->($scfg); my @paramsAddLun = ( '"','add ',$scstDiskName,$lun->{lun},'"', ">$targetPath/mgmt" ); my $resAddLun = $execute_command->($scfg, 'ssh', $timeout, 'echo', @paramsAddLun); do { my @paramsRemoveDevice = ( "\"del_device $scstDiskName\"", ">/sys/kernel/scst_tgt/handlers/vdisk_blockio/mgmt" ); my $resRemoveDevice = $execute_command->($scfg, 'ssh', $timeout, 'echo', @paramsRemoveDevice); $free_lu_name->($lun->{lun}); $update_config->($scfg); die "Could not create LUN: $lun->{lun} PATH: $lun->{Path} NAME: $scstDiskName: ".$resAddLun->{msg}; } unless $resAddLun->{result}; # make config persist here ? $update_config->($scfg); $clear_config->($scfg); return $resAddLun->{msg}; }; my $delete_lun = sub { my ($scfg, $timeout, $method, @params) = @_; my $res = {msg => undef}; $get_config->($scfg); my $path = $params[0]; my $targetPath; $targetPath = $get_target_path->($scfg); # my $tid = $get_target_tid->($scfg); # $debugmsg->('info',"Delete LUN $path"); foreach my $lun (@{$SETTINGS->{luns}}) { if ($lun->{Path} eq $path) { # $debugmsg->('info',"DO DELETE: $lun->{Path} $lun->{lun}"); # Delete LUN @params = ( "\"del $lun->{lun}\"", ">$targetPath/mgmt" ); $res = $execute_command->($scfg, 'ssh', $timeout, 'echo', @params); if ($res->{result}) { $debugmsg->('info',"[OK - delete LUN]: $lun->{Path} : $res->{msg} -> call: free_lu_name: $lun->{lun}"); # Delete Device my $scstDiskName = basename($lun->{Path}); @params = ( "\"del_device $scstDiskName\"", ">/sys/kernel/scst_tgt/handlers/vdisk_blockio/mgmt" ); $res = $execute_command->($scfg, 'ssh', $timeout, 'echo', @params); if ($res->{result}) { $free_lu_name->($lun->{lun}); $update_config->($scfg); $clear_config->($scfg); $get_config->($scfg); last; } else { die $res->{msg}; } } else { die $res->{msg}; } } } return $res->{msg}; }; my $import_lun = sub { my ($scfg, $timeout, $method, @params) = @_; return $create_lun->($scfg, $timeout, $method, @params); }; my $modify_lun = sub { my ($scfg, $timeout, $method, @params) = @_; my $res; my $scstDevice = basename($params[1]); @params = ('echo 1',">/sys/kernel/scst_tgt/devices/$scstDevice/resync_size"); $res = $execute_command->($scfg, 'ssh', $timeout, @params); die $res->{msg} unless $res->{result}; return $res->{msg}; }; my $add_view = sub { my ($scfg, $timeout, $method, @params) = @_; return ''; }; my $get_lun_cmd_map = sub { my ($method) = @_; my $cmdmap = { create_lu => { cmd => $create_lun }, delete_lu => { cmd => $delete_lun }, import_lu => { cmd => $import_lun }, modify_lu => { cmd => $modify_lun }, add_view => { cmd => $add_view }, list_view => { cmd => $list_view }, list_lu => { cmd => $list_lun }, }; die "unknown command '$method'" unless exists $cmdmap->{$method}; return $cmdmap->{$method}; }; sub run_lun_command { my ($scfg, $timeout, $method, @params) = @_; $get_config->($scfg); my $cmdmap = $get_lun_cmd_map->($method); my $msg = $cmdmap->{cmd}->($scfg, $timeout, $method, @params); return $msg; } sub get_base { return '/dev'; } 1;