# -*- coding: utf-8 -*-
###########################################################################
# Copyright (c), The AiiDA team. All rights reserved. #
# This file is part of the AiiDA code. #
# #
# The code is hosted on GitHub at https://github.com/aiidateam/aiida_core #
# For further information on the license, see the LICENSE.txt file #
# For further information please visit http://www.aiida.net #
###########################################################################
from stat import S_ISDIR,S_ISREG
import StringIO
import paramiko
import os
import glob
import aiida.transport
from aiida.common.utils import escape_for_bash
from aiida.transport import FileAttribute
from aiida.common import aiidalogger
# TODO : callback functions in paramiko are currently not used much and probably broken
def parse_sshconfig(computername):
config = paramiko.SSHConfig()
try:
config.parse(open(os.path.expanduser('~/.ssh/config')))
except IOError:
# No file found, so empty configuration
pass
return config.lookup(computername)
def convert_to_bool(string):
upstring = str(string).upper()
if upstring in ['Y', 'YES', 'T', 'TRUE']:
return True
elif upstring in ['N', 'NO', 'F', 'FALSE']:
return False
else:
raise ValueError("Invalid boolean value provided")
[docs]class SshTransport(aiida.transport.Transport):
"""
Support connection, command execution and data transfer to remote computers via SSH+SFTP.
"""
# Valid keywords accepted by the connect method of paramiko.SSHClient
# I disable 'password' and 'pkey' to avoid these data to get logged in the
# aiida log file.
_valid_connect_params = ['username', 'port', 'look_for_keys',
'key_filename', 'timeout', 'allow_agent',
'proxy_command', # Managed 'manually' in connect
'compress',
'gss_auth','gss_kex','gss_deleg_creds','gss_host', # for Kerberos support through python-gssapi
]
# Valid parameters for the ssh transport
# For each param, a class method with name
# _convert_PARAMNAME_fromstring
# should be defined, that returns the value converted from a string to
# a correct type, or raise a ValidationError
#
# moreover, if you want to help in the default configuration, you can
# define a _get_PARAMNAME_suggestion_string
# to return a suggestion; it must accept only one parameter, being a Computer
# instance
_valid_auth_params = _valid_connect_params + [
'load_system_host_keys',
'key_policy',
]
@classmethod
def _convert_username_fromstring(cls, string):
"""
Convert the username from string.
"""
return string
@classmethod
def _get_username_suggestion_string(cls, computer):
"""
Return a suggestion for the specific field.
"""
import getpass
config = parse_sshconfig(computer.hostname)
# Either the configured user in the .ssh/config, or the current username
return str(config.get('user',getpass.getuser()))
@classmethod
def _convert_port_fromstring(cls, string):
"""
Convert the port from string.
"""
from aiida.common.exceptions import ValidationError
try:
return int(string)
except ValueError:
raise ValidationError("The port must be an integer")
@classmethod
def _get_port_suggestion_string(cls, computer):
"""
Return a suggestion for the specific field.
"""
config = parse_sshconfig(computer.hostname)
# Either the configured user in the .ssh/config, or the default SSH port
return str(config.get('port',22))
@classmethod
def _convert_key_filename_fromstring(cls, string):
"""
Convert the port from string.
"""
from aiida.common.exceptions import ValidationError
path = os.path.expanduser(string)
if not os.path.isabs(path):
raise ValidationError("The key filename must be an absolute path")
if not os.path.exists(path):
raise ValidationError("The key filename must exist")
return path
@classmethod
def _get_key_filename_suggestion_string(cls, computer):
"""
Return a suggestion for the specific field.
"""
config = parse_sshconfig(computer.hostname)
try:
identities = config['identityfile']
# In paramiko > 0.10, identity file is a list of strings.
if isinstance(identities,basestring):
identity = identities
elif isinstance(identities,(list,tuple)):
if not identities:
# An empty list should not be provided; to be sure,
# anyway, behave as if no identityfile were defined
raise KeyError
# By default we suggest only the first one
identity = identities[0]
else:
# If the parser provides an unknown type, just skip to
# the 'except KeyError' section, as if no identityfile
# were provided (hopefully, this should never happen)
raise KeyError
except KeyError:
# No IdentityFile defined: return an empty string
return ""
return os.path.expanduser(identity)
@classmethod
def _convert_timeout_fromstring(cls, string):
"""
Convert the port from string.
"""
from aiida.common.exceptions import ValidationError
try:
return int(string)
except ValueError:
raise ValidationError("The timeout must be an integer")
@classmethod
def _get_timeout_suggestion_string(cls, computer):
"""
Return a suggestion for the specific field.
Provide 60s as a default timeout for connections.
"""
config = parse_sshconfig(computer.hostname)
return str(config.get('connecttimeout',"60"))
@classmethod
def _convert_allow_agent_fromstring(cls, string):
"""
Convert the port from string.
"""
from aiida.common.exceptions import ValidationError
try:
return convert_to_bool(string)
except ValueError:
raise ValidationError("Allow_agent must be an boolean")
@classmethod
def _get_allow_agent_suggestion_string(cls, computer):
"""
Return a suggestion for the specific field.
"""
return ""
@classmethod
def _convert_look_for_keys_fromstring(cls, string):
"""
Convert the port from string.
"""
from aiida.common.exceptions import ValidationError
try:
return convert_to_bool(string)
except ValueError:
raise ValidationError("look_for_keys must be an boolean")
@classmethod
def _get_look_for_keys_suggestion_string(cls, computer):
"""
Return a suggestion for the specific field.
"""
return ""
@classmethod
def _convert_proxy_command_fromstring(cls, string):
"""
Convert the proxy command from string.
"""
from aiida.common.exceptions import ValidationError
if str(string).strip():
return str(string)
else:
return None
@classmethod
def _get_proxy_command_suggestion_string(cls, computer):
"""
Return a suggestion for the specific field.
"""
config = parse_sshconfig(computer.hostname)
# Either the configured user in the .ssh/config, or the default SSH port
raw_string = str(config.get('proxycommand',''))
# Note: %h and %p get already automatically substituted with
# hostname and port by the config parser!
pieces = raw_string.split()
new_pieces = []
for piece in pieces:
if '>' in piece:
# If there is a piece with > to readdress stderr or stdout,
# skip from here on (anything else can only be readdressing)
break
else:
new_pieces.append(piece)
return " ".join(new_pieces)
@classmethod
def _convert_compress_fromstring(cls, string):
"""
Convert the port from string.
"""
from aiida.common.exceptions import ValidationError
try:
return convert_to_bool(string)
except ValueError:
raise ValidationError("compress must be an boolean")
@classmethod
def _get_compress_suggestion_string(cls, computer):
"""
Return a suggestion for the specific field.
"""
return "True"
@classmethod
def _convert_load_system_host_keys_fromstring(cls, string):
"""
Convert the port from string.
"""
from aiida.common.exceptions import ValidationError
try:
return convert_to_bool(string)
except ValueError:
raise ValidationError("compress must be an boolean")
@classmethod
def _get_load_system_host_keys_suggestion_string(cls, computer):
"""
Return a suggestion for the specific field.
"""
return "True"
@classmethod
def _convert_key_policy_fromstring(cls, string):
"""
Convert the port from string.
"""
return string
@classmethod
def _get_key_policy_suggestion_string(cls, computer):
"""
Return a suggestion for the specific field.
"""
return "RejectPolicy"
@classmethod
def _convert_gss_auth_fromstring(cls, string):
"""
Convert the gss auth. command from string.
"""
from aiida.common.exceptions import ValidationError
try:
return convert_to_bool(string)
except ValueError:
raise ValidationError("gss_auth must be an boolean")
@classmethod
def _get_gss_auth_suggestion_string(cls, computer):
"""
Return a suggestion for the specific field.
"""
config = parse_sshconfig(computer.hostname)
return str(config.get('gssapiauthentication',"no"))
@classmethod
def _convert_gss_kex_fromstring(cls, string):
"""
Convert the gss key exchange command from string.
"""
from aiida.common.exceptions import ValidationError
try:
return convert_to_bool(string)
except ValueError:
raise ValidationError("gss_kex must be an boolean")
@classmethod
def _get_gss_kex_suggestion_string(cls, computer):
"""
Return a suggestion for the specific field.
"""
config = parse_sshconfig(computer.hostname)
return str(config.get('gssapikeyexchange',"no"))
@classmethod
def _convert_gss_deleg_creds_fromstring(cls, string):
"""
Convert the gss auth. command from string.
"""
from aiida.common.exceptions import ValidationError
try:
return convert_to_bool(string)
except ValueError:
raise ValidationError("gss_deleg_creds must be an boolean")
@classmethod
def _get_gss_deleg_creds_suggestion_string(cls, computer):
"""
Return a suggestion for the specific field.
"""
config = parse_sshconfig(computer.hostname)
return str(config.get('gssapidelegatecredentials',"no"))
@classmethod
def _convert_gss_host_fromstring(cls, string):
"""
Convert the gss auth. command from string.
"""
from aiida.common.exceptions import ValidationError
return str(string)
@classmethod
def _get_gss_host_suggestion_string(cls, computer):
"""
Return a suggestion for the specific field.
"""
config = parse_sshconfig(computer.hostname)
return str(config.get('gssapihostname',computer.hostname))
def __init__(self, machine, **kwargs):
"""
Initialize the SshTransport class.
:param machine: the machine to connect to
:param load_system_host_keys: (optional, default False): if False, do not
load the system host keys
:param key_policy: (optional, default = paramiko.RejectPolicy()): the
policy to use for unknown keys
Other parameters valid for the ssh connect function (see the
self._valid_connect_params list) are passed to the connect
function (as port, username, password, ...); taken from the
accepted paramiko.SSHClient.connect() params)
"""
super(SshTransport,self).__init__()
self._is_open = False
self._sftp = None
self._machine = machine
self._client = paramiko.SSHClient()
self._load_system_host_keys = kwargs.pop(
'load_system_host_keys', False)
if self._load_system_host_keys:
self._client.load_system_host_keys()
self._missing_key_policy = kwargs.pop(
'key_policy','RejectPolicy') # This is paramiko default
if self._missing_key_policy == 'RejectPolicy':
self._client.set_missing_host_key_policy(paramiko.RejectPolicy())
elif self._missing_key_policy == 'WarningPolicy':
self._client.set_missing_host_key_policy(paramiko.WarningPolicy())
elif self._missing_key_policy == 'AutoAddPolicy':
self._client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
else:
raise ValueError("Unknown value of the key policy, allowed values "
"are: RejectPolicy, WarningPolicy, AutoAddPolicy")
self._connect_args = {}
for k in self._valid_connect_params:
try:
self._connect_args[k] = kwargs.pop(k)
except KeyError:
pass
if kwargs:
raise ValueError("The following parameters were not accepted by "
"the transport: {}".format(
",".join(str(k) for k in kwargs)))
[docs] def open(self):
"""
Open a SSHClient to the machine possibly using the parameters given
in the __init__.
Also opens a sftp channel, ready to be used.
The current working directory is set explicitly, so it is not None.
:raise InvalidOperation: if the channel is already open
"""
from aiida.common.exceptions import InvalidOperation
if self._is_open:
raise InvalidOperation("Cannot open the transport twice")
# Open a SSHClient
connection_arguments = self._connect_args
proxystring = connection_arguments.pop('proxy_command', None)
if proxystring is not None:
proxy = paramiko.ProxyCommand(proxystring)
connection_arguments['sock'] = proxy
try:
self._client.connect(self._machine,**connection_arguments)
except Exception as e:
self.logger.error("Error connecting through SSH: [{}] {}, "
"connect_args were: {}".format(
e.__class__.__name__,
e.message,
self._connect_args))
raise
# Open also a SFTPClient
self._sftp = self._client.open_sftp()
# Set the current directory to a explicit path, and not to None
self._sftp.chdir(self._sftp.normalize('.'))
self._is_open = True
return self
[docs] def close(self):
"""
Close the SFTP channel, and the SSHClient.
:todo: correctly manage exceptions
:raise InvalidOperation: if the channel is already open
"""
from aiida.common.exceptions import InvalidOperation
if not self._is_open:
raise InvalidOperation("Cannot close the transport: "
"it is already closed")
self._sftp.close()
self._client.close()
self._is_open = False
@property
def sshclient(self):
if not self._is_open:
raise aiida.transport.TransportInternalError(
"Error, ssh method called for SshTransport "
"without opening the channel first")
return self._client
@property
def sftp(self):
if not self._is_open:
raise aiida.transport.TransportInternalError(
"Error, sftp method called for SshTransport "
"without opening the channel first")
return self._sftp
def __str__(self):
"""
Return a useful string.
"""
conn_info = self._machine
try:
conn_info = "{}@{}".format(self._connect_args['username'],
conn_info)
except KeyError:
# No username explicitly defined: ignore
pass
try:
conn_info += ':{}'.format(self._connect_args['port'])
except KeyError:
# No port explicitly defined: ignore
pass
return "{} [{}]".format("OPEN" if self._is_open else "CLOSED",
conn_info)
[docs] def chdir(self, path):
"""
Change directory of the SFTP session. Emulated internally by
paramiko.
Differently from paramiko, if you pass None to chdir, nothing
happens and the cwd is unchanged.
"""
from paramiko.sftp import SFTPError
old_path = self.sftp.getcwd()
if path is not None:
try:
self.sftp.chdir(path)
except SFTPError as e:
# e.args[0] is an error code. For instance,
# 20 is 'the object is not a directory'
# Here I just re-raise the message as IOError
raise IOError(e.args[1])
# Paramiko already checked that path is a folder, otherwise I would
# have gotten an exception. Now, I want to check that I have read
# permissions in this folder (nothing is said on write permissions,
# though).
# Otherwise, if I do _exec_command_internal, that as a first operation
# cd's in a folder, I get a wrong retval, that is an unwanted behavior.
#
# Note: I don't store the result of the function; if I have no
# read permissions, this will raise an exception.
try:
self.sftp.stat('.')
except IOError as e:
if 'Permission denied' in e:
self.chdir(old_path)
raise IOError(e)
[docs] def normalize(self, path):
"""
Returns the normalized path (removing double slashes, etc...)
"""
return self.sftp.normalize(path)
[docs] def getcwd(self):
"""
Return the current working directory for this SFTP session, as
emulated by paramiko. If no directory has been set with chdir,
this method will return None. But in __enter__ this is set explicitly,
so this should never happen within this class.
"""
return self.sftp.getcwd()
[docs] def makedirs(self,path,ignore_existing=False):
"""
Super-mkdir; create a leaf directory and all intermediate ones.
Works like mkdir, except that any intermediate path segment (not
just the rightmost) will be created if it does not exist.
NOTE: since os.path.split uses the separators as the host system
(that could be windows), I assume the remote computer is Linux-based
and use '/' as separators!
:param path: directory to create (string)
:param ignore_existing: if set to true, it doesn't give any error
if the leaf directory does already exist (bool)
:raise OSError: If the directory already exists.
"""
# check to avoid creation of empty dirs
path = os.path.normpath(path)
if path.startswith('/'):
to_create = path.strip().split('/')[1:]
this_dir='/'
else:
to_create = path.strip().split('/')
this_dir = ''
for count,element in enumerate(to_create):
if count>0:
this_dir += '/'
this_dir += element
if count+1==len(to_create) and self.isdir(this_dir) and ignore_existing:
return
if count+1==len(to_create) and self.isdir(this_dir) and not ignore_existing:
self.mkdir(this_dir)
if not self.isdir(this_dir):
self.mkdir(this_dir)
[docs] def mkdir(self,path,ignore_existing=False):
"""
Create a folder (directory) named path.
:param path: name of the folder to create
:param ignore_existing: if True, does not give any error if the directory
already exists
:raise OSError: If the directory already exists.
"""
if ignore_existing and self.isdir(path):
return
try:
self.sftp.mkdir(path)
except IOError as e:
if os.path.isabs(path):
raise OSError(
"Error during mkdir of '{}', "
"maybe you don't have the permissions to do it, "
"or the directory already exists? ({})".format(
path, e.message))
else:
raise OSError(
"Error during mkdir of '{}' from folder '{}', "
"maybe you don't have the permissions to do it, "
"or the directory already exists? ({})".format(
path, self.getcwd(), e.message))
# TODO : implement rmtree
[docs] def rmtree(self,path):
"""
Remove a file or a directory at path, recursively
Flags used: -r: recursive copy; -f: force, makes the command non interactive;
:param path: remote path to delete
:raise IOError: if the rm execution failed.
"""
# Assuming linux rm command!
# TODO : do we need to avoid the aliases when calling rm_exe='rm'? Call directly /bin/rm?
rm_exe = 'rm'
rm_flags = '-r -f'
# if in input I give an invalid object raise ValueError
if not path:
raise ValueError('Input to rmtree() must be a non empty string. ' +
'Found instead %s as path' % path)
command = '{} {} {}'.format(rm_exe,
rm_flags,
escape_for_bash(path))
retval,stdout,stderr = self.exec_command_wait(command)
if retval == 0:
if stderr.strip():
self.logger.warning("There was nonempty stderr in the rm "
"command: {}".format(stderr))
return True
else:
self.logger.error("Problem executing rm. Exit code: {}, stdout: '{}', "
"stderr: '{}'".format(retval, stdout, stderr))
raise IOError("Error while executing rm. Exit code: {}".format(retval) )
[docs] def rmdir(self, path):
"""
Remove the folder named 'path' if empty.
"""
self.sftp.rmdir(path)
[docs] def isdir(self,path):
"""
Return True if the given path is a directory, False otherwise.
Return False also if the path does not exist.
"""
# Return False on empty string (paramiko would map this to the local
# folder instead)
if not path:
return False
try:
return S_ISDIR(self.sftp.stat(path).st_mode)
except IOError as e:
if getattr(e,"errno",None) == 2:
# errno=2 means path does not exist: I return False
return False
else:
raise # Typically if I don't have permissions (errno=13)
[docs] def chmod(self,path,mode):
"""
Change permissions to path
:param path: path to file
:param mode: new permission bits (integer)
"""
if not path:
raise IOError("Input path is an empty argument.")
return self.sftp.chmod(path,mode)
def _os_path_split_asunder(self, path):
"""
Used by makedirs. Takes path (a str)
and returns a list deconcatenating the path
"""
parts = []
while True:
newpath, tail = os.path.split(path)
if newpath == path:
assert not tail
if path: parts.append(path)
break
parts.append(tail)
path = newpath
parts.reverse()
return parts
[docs] def put(self,localpath,remotepath,callback=None,dereference=True,overwrite=True,
ignore_nonexisting=False):
"""
Put a file or a folder from local to remote.
Redirects to putfile or puttree.
:param localpath: an (absolute) local path
:param remotepath: a remote path
:param dereference: follow symbolic links (boolean).
Default = True (default behaviour in paramiko). False is not implemented.
:param overwrite: if True overwrites files and folders (boolean).
Default = False.
:raise ValueError: if local path is invalid
:raise OSError: if the localpath does not exist
"""
# TODO: flag confirm exists since v1.7.7. What is the paramiko
# version supported?
# TODO : add dereference
if not dereference:
raise NotImplementedError
if not os.path.isabs(localpath):
raise ValueError("The localpath must be an absolute path")
if self.has_magic(localpath):
if self.has_magic(remotepath):
raise ValueError("Pathname patterns are not allowed in the "
"destination")
# use the imported glob to analyze the path locally
to_copy_list = glob.glob(localpath)
rename_remote = False
if len(to_copy_list)>1:
# I can't scp more than one file on a single file
if self.isfile(remotepath):
raise OSError("Remote destination is not a directory")
# I can't scp more than one file in a non existing directory
elif not self.path_exists(remotepath): # questo dovrebbe valere solo per file
raise OSError("Remote directory does not exist")
else: # the remote path is a directory
rename_remote=True
for s in to_copy_list:
if os.path.isfile(s):
if rename_remote: # copying more than one file in one directory
# here is the case isfile and more than one file
r = os.path.join(remotepath,os.path.split(s)[1])
self.putfile(s,r,callback,dereference,overwrite )
elif self.isdir(remotepath): # one file to copy in '.'
r = os.path.join(remotepath,os.path.split(s)[1])
self.putfile(s,r,callback,dereference,overwrite )
else: # one file to copy on one file
self.putfile(s,remotepath,callback,dereference,overwrite )
else:
self.puttree(s,remotepath,callback,dereference,overwrite )
else:
if os.path.isdir(localpath):
self.puttree( localpath,remotepath,callback,dereference,overwrite )
elif os.path.isfile(localpath):
if self.isdir(remotepath):
r = os.path.join(remotepath,os.path.split(localpath)[1])
self.putfile( localpath,r,callback,dereference,overwrite )
else:
self.putfile( localpath,remotepath,callback,dereference,overwrite )
else:
if ignore_nonexisting:
pass
else:
raise OSError("The local path {} does not exist"
.format(localpath))
[docs] def putfile(self,localpath,remotepath,callback=None,dereference=True,overwrite=True):
"""
Put a file from local to remote.
:param localpath: an (absolute) local path
:param remotepath: a remote path
:param overwrite: if True overwrites files and folders (boolean).
Default = True.
:raise ValueError: if local path is invalid
:raise OSError: if the localpath does not exist,
or unintentionally overwriting
"""
# TODO : add dereference
if not dereference:
raise NotImplementedError
# TODO : check what happens if I give in input a directory
if not os.path.isabs(localpath):
raise ValueError("The localpath must be an absolute path")
if self.isfile(remotepath) and not overwrite:
raise OSError('Destination already exists: not overwriting it')
return self.sftp.put(localpath,remotepath,callback=callback)
[docs] def puttree(self,localpath,remotepath,callback=None,dereference=True,overwrite=True): # by default overwrite
"""
Put a folder recursively from local to remote.
:param localpath: an (absolute) local path
:param remotepath: a remote path
:param dereference: follow symbolic links (boolean)
Default = True (default behaviour in paramiko). False is not implemented.
:param overwrite: if True overwrites files and folders (boolean).
Default = True
:raise ValueError: if local path is invalid
:raise OSError: if the localpath does not exist, or trying to overwrite
:raise IOError: if remotepath is invalid
.. note:: setting dereference equal to True could cause infinite loops.
see os.walk() documentation
"""
# TODO : add dereference
if not dereference:
raise NotImplementedError
if not os.path.isabs(localpath):
raise ValueError("The localpath must be an absolute path")
if not os.path.exists(localpath):
raise OSError("The localpath does not exists")
if not os.path.isdir(localpath):
raise ValueError("Input localpath is not a folder: {}".format(localpath))
if not remotepath:
raise IOError("remotepath must be a non empty string")
if self.path_exists(remotepath) and not overwrite:
raise OSError("Can't overwrite existing files")
if self.isfile(remotepath):
raise OSError("Cannot copy a directory into a file")
if not self.isdir(remotepath): # in this case copy things in the remotepath directly
self.mkdir(remotepath) # and make a directory at its place
else: # remotepath exists already: copy the folder inside of it!
remotepath = os.path.join(remotepath,os.path.split(localpath)[1])
self.mkdir(remotepath) # create a nested folder
# TODO, NOTE: we are not using 'onerror' because we checked above that
# the folder exists, but it would be better to use it
for this_source in os.walk(localpath):
# Get the relative path
this_basename = os.path.relpath(path=this_source[0],
start=localpath)
try:
self.sftp.stat( os.path.join(remotepath,this_basename) )
except IOError as e:
import errno
if e.errno == errno.ENOENT: # Missing file
self.mkdir( os.path.join(remotepath,this_basename) )
else:
raise
for this_file in this_source[2]:
this_local_file = os.path.join(localpath,this_basename,this_file)
this_remote_file = os.path.join(remotepath,this_basename,this_file)
self.putfile(this_local_file,this_remote_file)
[docs] def get(self,remotepath,localpath,callback=None,dereference=True,overwrite=True,
ignore_nonexisting=False):
"""
Get a file or folder from remote to local.
Redirects to getfile or gettree.
:param remotepath: a remote path
:param localpath: an (absolute) local path
:param dereference: follow symbolic links.
Default = True (default behaviour in paramiko).
False is not implemented.
:param overwrite: if True overwrites files and folders.
Default = False
:raise ValueError: if local path is invalid
:raise IOError: if the remotepath is not found
"""
# TODO : add dereference
if not dereference:
raise NotImplementedError
if not os.path.isabs(localpath):
raise ValueError("The localpath must be an absolute path")
if self.has_magic(remotepath):
if self.has_magic(localpath):
raise ValueError("Pathname patterns are not allowed in the "
"destination")
# use the self glob to analyze the path remotely
to_copy_list = self.glob(remotepath)
rename_local = False
if len(to_copy_list)>1:
# I can't scp more than one file on a single file
if os.path.isfile(localpath):
raise IOError("Remote destination is not a directory")
# I can't scp more than one file in a non existing directory
elif not os.path.exists(localpath): # this should hold only for files
raise OSError("Remote directory does not exist")
else: # the remote path is a directory
rename_local=True
for s in to_copy_list:
if self.isfile(s):
if rename_local: # copying more than one file in one directory
# here is the case isfile and more than one file
r = os.path.join(localpath,os.path.split(s)[1])
self.getfile(s,r,callback,dereference,overwrite )
else: # one file to copy on one file
self.getfile(s,localpath,callback,dereference,overwrite )
else:
self.gettree(s,localpath,callback,dereference,overwrite )
else:
if self.isdir(remotepath):
self.gettree( remotepath,localpath,callback,dereference,overwrite )
elif self.isfile(remotepath):
if os.path.isdir(localpath):
r = os.path.join(localpath,os.path.split(remotepath)[1])
self.getfile( remotepath,r,callback,dereference,overwrite )
else:
self.getfile( remotepath,localpath,callback,dereference,overwrite )
else:
if ignore_nonexisting:
pass
else:
raise IOError("The remote path {} does not exist"
.format(remotepath))
[docs] def getfile(self,remotepath,localpath,callback=None,dereference=True,overwrite=True):
"""
Get a file from remote to local.
:param remotepath: a remote path
:param localpath: an (absolute) local path
:param overwrite: if True overwrites files and folders.
Default = False
:raise ValueError: if local path is invalid
:raise OSError: if unintentionally overwriting
"""
# TODO : add dereference
if not os.path.isabs(localpath):
raise ValueError("localpath must be an absolute path")
if os.path.isfile(localpath) and not overwrite:
raise OSError('Destination already exists: not overwriting it')
if not dereference:
raise NotImplementedError
# Workaround for bug #724 in paramiko -- remove localpath on IOError
try:
return self.sftp.get(remotepath,localpath,callback)
except IOError as e:
try:
os.remove(localpath)
except OSError:
pass
raise
[docs] def gettree(self,remotepath,localpath,callback=None,dereference=True,overwrite=True):
"""
Get a folder recursively from remote to local.
:param remotepath: a remote path
:param localpath: an (absolute) local path
:param dereference: follow symbolic links.
Default = True (default behaviour in paramiko).
False is not implemented.
:param overwrite: if True overwrites files and folders.
Default = False
:raise ValueError: if local path is invalid
:raise IOError: if the remotepath is not found
:raise OSError: if unintentionally overwriting
"""
# TODO : add dereference
if not dereference:
raise NotImplementedError
if not remotepath:
raise IOError("Remotepath must be a non empty string")
if not localpath:
raise ValueError("Localpaths must be a non empty string")
if not os.path.isabs(localpath):
raise ValueError("Localpaths must be an absolute path")
if not self.isdir(remotepath):
raise IOError("Input remotepath is not a folder: {}".format(localpath))
if os.path.exists(localpath) and not overwrite:
raise OSError("Can't overwrite existing files")
if os.path.isfile(localpath):
raise OSError("Cannot copy a directory into a file")
if not os.path.isdir(localpath): # in this case copy things in the remotepath directly
os.mkdir(localpath) # and make a directory at its place
else: # localpath exists already: copy the folder inside of it!
localpath = os.path.join(localpath,os.path.split(remotepath)[1])
os.mkdir(localpath) # create a nested folder
item_list = self.listdir(remotepath)
dest = str(localpath)
for item in item_list:
item = str(item)
if self.isdir( os.path.join(remotepath,item) ):
self.gettree( os.path.join(remotepath,item) , os.path.join(dest,item) )
else:
self.getfile( os.path.join(remotepath,item) , os.path.join(dest,item) )
[docs] def get_attribute(self,path):
"""
Returns the object Fileattribute, specified in aiida.transport
Receives in input the path of a given file.
"""
paramiko_attr = self.sftp.lstat(path)
aiida_attr = FileAttribute()
# map the paramiko class into the aiida one
# note that paramiko object contains more informations than the aiida
for key in aiida_attr._valid_fields:
aiida_attr[key] = getattr(paramiko_attr,key)
return aiida_attr
[docs] def copyfile(self,remotesource,remotedestination,dereference=False,
pattern=None):
"""
Copy a file from remote source to remote destination
Redirects to copy().
:param remotesource:
:param remotedestination:
:param dereference:
:param pattern:
"""
cp_flags = '-f'
return self.copy(remotesource,remotedestination,dereference,cp_flags,pattern)
[docs] def copytree(self,remotesource,remotedestination,dereference=False,
pattern=None):
"""
copy a folder recursively from remote source to remote destination
Redirects to copy()
:param remotesource:
:param remotedestination:
:param dereference:
:param pattern:
"""
cp_flags = '-r -f'
return self.copy(remotesource,remotedestination,dereference,cp_flags,pattern)
[docs] def copy(self,remotesource,remotedestination,dereference=False):
"""
Copy a file or a directory from remote source to remote destination.
Flags used: ``-r``: recursive copy; ``-f``: force, makes the command non interactive;
``-L`` follows symbolic links
:param remotesource: file to copy from
:param remotedestination: file to copy to
:param dereference: if True, copy content instead of copying the symlinks only
Default = False.
:raise IOError: if the cp execution failed.
.. note:: setting dereference equal to True could cause infinite loops.
"""
# In the majority of cases, we should deal with linux cp commands
# TODO : do we need to avoid the aliases when calling cp_exe='cp'? Call directly /bin/cp?
# TODO: verify that it does not re
# For the moment, these are hardcoded. They may become parameters
# as soon as we see the need.
cp_flags='-r -f'
cp_exe='cp'
## To evaluate if we also want -p: preserves mode,ownership and timestamp
if dereference:
# use -L; --dereference is not supported on mac
cp_flags+=' -L'
# if in input I give an invalid object raise ValueError
if not remotesource:
raise ValueError('Input to copy() must be a non empty string. ' +
'Found instead %s as remotesource' % remotesource)
if not remotedestination:
raise ValueError('Input to copy() must be a non empty string. ' +
'Found instead %s as remotedestination' % remotedestination)
if self.has_magic(remotedestination):
raise ValueError("Pathname patterns are not allowed in the "
"destination")
if self.has_magic(remotesource):
to_copy_list = self.glob(remotesource)
if len(to_copy_list)>1:
if not self.path_exists(remotedestination) or self.isfile(remotedestination):
raise OSError("Can't copy more than one file in the same "
"destination file")
for s in to_copy_list:
self._exec_cp(cp_exe,cp_flags,s,remotedestination)
else:
self._exec_cp(cp_exe,cp_flags,remotesource,remotedestination)
def _exec_cp(self,cp_exe,cp_flags,src,dst):
# to simplify writing the above copy function
command = '{} {} {} {}'.format(cp_exe,cp_flags,escape_for_bash(src),
escape_for_bash(dst))
retval,stdout,stderr = self.exec_command_wait(command)
# TODO : check and fix below
if retval == 0:
if stderr.strip():
self.logger.warning("There was nonempty stderr in the cp "
"command: {}".format(stderr))
else:
self.logger.error("Problem executing cp. Exit code: {}, stdout: '{}', "
"stderr: '{}', command: '{}'"
.format(retval, stdout, stderr,command))
raise IOError("Error while executing cp. Exit code: {}, "
"stdout: '{}', stderr: '{}', "
"command: '{}'".format(retval, stdout, stderr,
command) )
def _local_listdir(self,path,pattern=None):
"""
Acts on the local folder, for the rest, same as listdir
"""
if not pattern:
return os.listdir( path )
else:
import re
if path.startswith('/'): # always this is the case in the local case
base_dir = path
else:
base_dir = os.path.join(os.getcwd(),path)
filtered_list = glob.glob(os.path.join(base_dir,pattern) )
if not base_dir.endswith(os.sep):
base_dir+=os.sep
return [ re.sub(base_dir,'',i) for i in filtered_list ]
[docs] def listdir(self,path='.',pattern=None):
"""
Get the list of files at path.
:param path: default = '.'
:param pattern: returns the list of files matching pattern.
Unix only. (Use to emulate ``ls *`` for example)
"""
if not pattern:
return self.sftp.listdir(path)
else:
import re
if path.startswith('/'):
base_dir = path
else:
base_dir = os.path.join(self.getcwd(),path)
filtered_list = glob.glob(os.path.join(base_dir,pattern) )
if not base_dir.endswith('/'):
base_dir += '/'
return [ re.sub(base_dir,'',i) for i in filtered_list ]
[docs] def remove(self,path):
"""
Remove a single file at 'path'
"""
return self.sftp.remove(path)
[docs] def rename(self,src,dst):
"""
Rename a file or folder from src to dst.
:param str oldpath: existing name of the file or folder
:param str newpath: new name for the file or folder
:raises IOError: if src/dst is not found
:raises ValueError: if src/dst is not a valid string
"""
if not src:
raise ValueError("Source {} is not a valid string".format(src))
if not dst:
raise ValueError("Destination {} is not a valid string".format(dst))
if not self.isfile( src ):
if not self.isdir( src ):
raise IOError("Source {} does not exist".format(src))
if not self.isfile( dst ):
if not self.isdir( dst ):
raise IOError("Destination {} does not exist".format(dst))
return self.sftp.rename(src,dst)
[docs] def isfile(self,path):
"""
Return True if the given path is a file, False otherwise.
Return False also if the path does not exist.
"""
# This should not be needed for files, since an empty string should
# be mapped by paramiko to the local directory - which is not a file -
# but this is just to be sure
if not path:
return False
try:
self.logger.debug("stat for path '{}' ('{}'): {} [{}]".format(
path, self.sftp.normalize(path),
self.sftp.stat(path),self.sftp.stat(path).st_mode))
return S_ISREG(self.sftp.stat(path).st_mode)
except IOError as e:
if getattr(e,"errno",None) == 2:
# errno=2 means path does not exist: I return False
return False
else:
raise # Typically if I don't have permissions (errno=13)
def _exec_command_internal(self,command,combine_stderr=False,bufsize=-1):
"""
Executes the specified command, first changing directory to the
current working directory are returned by
self.getcwd().
Does not wait for the calculation to finish.
For a higher-level _exec_command_internal that automatically waits for the
job to finish, use exec_command_wait.
:param command: the command to execute
:param combine_stderr: (default False) if True, combine stdout and
stderr on the same buffer (i.e., stdout).
Note: If combine_stderr is True, stderr will always be empty.
:param bufsize: same meaning of the one used by paramiko.
:return: a tuple with (stdin, stdout, stderr, channel),
where stdin, stdout and stderr behave as file-like objects,
plus the methods provided by paramiko, and channel is a
paramiko.Channel object.
"""
channel = self.sshclient.get_transport().open_session()
channel.set_combine_stderr(combine_stderr)
if self.getcwd() is not None:
escaped_folder = escape_for_bash(self.getcwd())
command_to_execute = ("cd {escaped_folder} && "
"{real_command}".format(
escaped_folder = escaped_folder,
real_command = command))
else:
command_to_execute = command
self.logger.debug("Command to be executed: {}".format(
command_to_execute))
channel.exec_command(command_to_execute)
stdin = channel.makefile('wb',bufsize)
stdout = channel.makefile('rb',bufsize)
stderr = channel.makefile_stderr('rb',bufsize)
return stdin, stdout, stderr, channel
[docs] def exec_command_wait(self,command,stdin=None,combine_stderr=False,
bufsize=-1):
"""
Executes the specified command and waits for it to finish.
:param command: the command to execute
:param stdin: (optional,default=None) can be a string or a
file-like object.
:param combine_stderr: (optional, default=False) see docstring of
self._exec_command_internal()
:param bufsize: same meaning of paramiko.
:return: a tuple with (return_value, stdout, stderr) where stdout and stderr
are strings.
"""
#TODO: To see if like this it works or hangs because of buffer problems.
ssh_stdin, stdout, stderr, channel = self._exec_command_internal(
command, combine_stderr,bufsize=bufsize)
if stdin is not None:
if isinstance(stdin, basestring):
filelike_stdin = StringIO.StringIO(stdin)
else:
filelike_stdin = stdin
try:
for l in filelike_stdin.readlines():
ssh_stdin.write(l)
except AttributeError:
raise ValueError("stdin can only be either a string of a "
"file-like object!")
# I flush and close them anyway; important to call shutdown_write
# to avoid hangouts
ssh_stdin.flush()
ssh_stdin.channel.shutdown_write()
output_text = stdout.read()
# I get the return code (blocking, but in principle I waited above)
retval = channel.recv_exit_status()
stderr_text = stderr.read()
return retval, output_text, stderr_text
[docs] def gotocomputer_command(self, remotedir):
"""
Specific gotocomputer string to connect to a given remote computer via
ssh and directly go to the calculation folder.
"""
#TODO: add also ProxyCommand and Timeout support
further_params = []
if 'username' in self._connect_args:
further_params.append("-l {}".format(escape_for_bash(
self._connect_args['username'])))
if 'port' in self._connect_args:
further_params.append("-p {}".format(self._connect_args['port']))
if 'key_filename' in self._connect_args:
further_params.append("-i {}".format(escape_for_bash(
self._connect_args['key_filename'])))
further_params_str = " ".join(further_params)
connect_string = """ssh -t {machine} {further_params} "if [ -d {escaped_remotedir} ] ; then cd {escaped_remotedir} ; bash -l ; else echo ' ** The directory' ; echo ' ** {remotedir}' ; echo ' ** seems to have been deleted, I logout...' ; fi" """.format(
further_params=further_params_str,
machine=self._machine,
escaped_remotedir="'{}'".format(remotedir),
remotedir=remotedir)
# print connect_string
return connect_string
#return """ssh -Y -t {machine} "if [ -d {escaped_remotedir} ] ; then cd {escaped_remotedir} ; bash -c "{bash_call_escaped}" ; else echo ' ** The directory' ; echo ' ** {remotedir}' ; echo ' ** seems to have been deleted, I logout...' ; fi" """.format(
# machine=self._machine, escaped_remotedir="'{}'".format(remotedir), remotedir=remotedir,
# bash_call_escaped=escape_for_bash("""bash --rcfile <(echo 'if [ -e ~/.bashrc ] ; then source ~/.bashrc ; fi ; export PS1="\[\033[01;31m\][AiiDA]\033[00m\]$PS1"')"""))
[docs] def symlink(self,remotesource,remotedestination):
"""
Create a symbolic link between the remote source and the remote
destination.
:param remotesource: remote source. Can contain a pattern.
:param remotedestination: remote destination
"""
# paramiko gives some errors if path is starting with '.'
s = os.path.normpath(remotesource)
d = os.path.normpath(remotedestination)
if self.has_magic(s):
if self.has_magic(d):
# if there are patterns in dest, I don't know which name to assign
raise ValueError("Remotedestination cannot have patterns")
# find all files matching pattern
for this_s in self.glob(s):
# create the name of the link: take the last part of the path
this_d = os.path.join(remotedestination,
os.path.split(this_s)[-1])
self.sftp.symlink(this_s, this_d)
else:
self.sftp.symlink(s,d)
[docs] def path_exists(self,path):
"""
Check if path exists
"""
import errno
try:
self.sftp.stat(path)
except IOError, e:
if e.errno == errno.ENOENT:
return False
raise
else:
return True