#!/usr/bin/env python3
"""
The apssh binary makes a rather straightforward use of the library,
except maybe for the way it handles the fuzzy notion of targets,
that can be defined as either hostnames directly, or from files that contain
hostnames, or from directories that contain files named after hostnames.
"""
# allow catch-all exceptions
# pylint: disable=W0703
# for using a locally cloned branch of asyncssh
# import sys
# sys.path.insert(0, "../../asyncssh/")
import os
from pathlib import Path
import argparse
from asynciojobs import Scheduler
from .util import print_stderr
from .config import (default_time_name, default_timeout, default_username,
default_remote_workdir, local_config_dir)
from .sshproxy import SshProxy
from .formatters import (RawFormatter, ColonFormatter,
TimeColonFormatter, SubdirFormatter,
TerminalFormatter)
from .keys import load_private_keys
from .version import __version__ as apssh_version
from .sshjob import SshJob
from .commands import Run, RunScript, RunString
[docs]class Apssh:
"""
Main class for apssh utility
"""
def __init__(self):
self.proxies = []
self.formatter = None
#
self.parser = None
self.parsed_args = None
self.loaded_private_keys = None
def __repr__(self):
return "".join([str(p) for p in self.proxies])
# returns a valid Path object, or None
@staticmethod
def locate_file(file): # pylint: disable=C0111
path = Path(file)
if path.exists():
return path
if not path.is_absolute():
in_home = local_config_dir / path
# somehow pylints figured in_home is a PurePath
if in_home.exists(): # pylint: disable=E1101
return in_home
return None
[docs] def analyze_target(self, target):
"""
This function is used to guess the meaning of all the targets passed
to the ``apssh`` command through its ``-t/--target`` option.
Parameters:
target: a string passed to ``--target``
Returns:
(bool, list): a tuple, whose meaning is described below.
A target can be specified as either
* a **filename**. If it is a relative filename it is also searched
in ``~/.apssh``. If an existing file can be located this way,
and it can be parsed, then the returned object will be of the form::
True, [ hostname1, ...]
* a **directory name**. Here again the search is also done in
``~/.apssh``. If an existing directory can be found,
all the simple files that are found immediately under the
specified directory are taken as hostnames, and in this case
``analyze_target`` returns::
True, [ hostname1, ...]
This is notably for use together with the ``--mark`` option,
so that one can easily select reachable nodes only,
or just as easily exclude failing nodes.
* otherwise, the incoming target is then expected to be a string that
directly contains the hostnames, and so it is simply split along
white spaces, and the return code is then::
True, [ hostname1, ...]
* If anything goes wrong, return code is::
False, []
for example, this is the case when the file exists
but cannot be parsed - in which case it is probably not a hostname.
"""
names = []
# located is a Path object - or None
located = self.locate_file(target)
if located:
if located.is_dir():
# directory
onlyfiles = [f for f in os.listdir(target)
if (located / f).is_file()]
return True, onlyfiles
else:
# file
try:
with located.open() as inputfile:
for line in inputfile:
line = line.strip()
if line.startswith('#'):
continue
line = line.split()
for token in line:
names += token.split()
return True, names
except FileNotFoundError as exc:
return False, None
except Exception as exc:
print_stderr(
"Unexpected exception when parsing file {}, {}"
.format(target, exc))
if self.parsed_args.debug:
import traceback
traceback.print_exc()
return False, None
else:
# string
return True, target.split()
# create tuples username, hostname
# use the username if already present in the target,
# otherwise the one specified with --username
def user_host(self, target): # pylint: disable=C0111
try:
user, hostname = target.split('@')
return user, hostname
except Exception:
return self.parsed_args.login, target
def create_proxies(self, gateway): # pylint: disable=C0111
# start with parsing excludes if any
excludes = set()
for exclude in self.parsed_args.excludes:
parsed, cli_excludes = self.analyze_target(exclude)
excludes.update(cli_excludes)
if self.parsed_args.dry_run:
print("========== {} excludes found:".format(len(excludes)))
for exclude in excludes:
print(exclude)
# gather targets as mentioned in -t -x args
hostnames = []
actually_excluded = 0
# read input file(s)
for target in self.parsed_args.targets:
parsed, cli_targets = self.analyze_target(target)
if not parsed:
print("WARNING: ignoring target {}".format(target))
continue
for target2 in cli_targets:
if target2 not in excludes:
hostnames.append(target2)
else:
actually_excluded += 1
if not hostnames:
print("it makes no sense to run apssh without any hostname")
self.parser.print_help()
exit(1)
if self.parsed_args.dry_run:
print("========== {} hostnames selected ({} excluded):"
.format(len(hostnames), actually_excluded))
for hostname in hostnames:
print(hostname)
exit(0)
# create proxies
self.proxies = [SshProxy(hostname, username=username,
keys=self.loaded_private_keys,
gateway=gateway,
formatter=self.get_formatter(),
timeout=self.parsed_args.timeout,
debug=self.parsed_args.debug)
for username, hostname in (self.user_host(target)
for target in hostnames)] # pylint: disable=c0330
return self.proxies
def get_formatter(self): # pylint: disable=C0111
if self.formatter is None:
verbose = self.parsed_args.verbose
if self.parsed_args.format:
self.formatter = TerminalFormatter(
self.parsed_args.format, verbose=verbose)
elif self.parsed_args.raw_format:
self.formatter = RawFormatter(verbose=verbose)
elif self.parsed_args.time_colon_format:
self.formatter = TimeColonFormatter(verbose=verbose)
elif self.parsed_args.date_time:
run_name = default_time_name
self.formatter = SubdirFormatter(run_name, verbose=verbose)
elif self.parsed_args.out_dir:
self.formatter = SubdirFormatter(self.parsed_args.out_dir,
verbose=verbose)
else:
self.formatter = ColonFormatter(verbose=verbose)
return self.formatter
def main(self, *test_argv): # pylint: disable=r0915,r0912,r0914,c0111
self.parser = parser = argparse.ArgumentParser()
# scope - on what hosts
parser.add_argument(
"-s", "--script", action='store_true', default=False,
help="""If this flag is present, the first element of the remote
command is assumed to be either the name of a local script, or,
if this is not found, the body of a local script, that will be
copied over before being executed remotely.
In this case it should be executable.
On the remote boxes it will be installed
and run in the {} directory.
""".format(default_remote_workdir))
parser.add_argument(
"-i", "--includes", dest='includes', default=[], action='append',
help="""for script mode only : a list of local files that are
pushed remotely together with the local script,
and in the same location; useful when you want to
to run remotely a shell script that sources other files;
remember that on the remote end all files (scripts and includes)
end up in the same location""")
parser.add_argument(
"-t", "--target", dest='targets', action='append', default=[],
help="""
specify targets (additive); at least one is required;
each target can be either
* a space-separated list of hostnames
* the name of a file containing hostnames
* the name of a directory containing files named after hostnames;
see e.g. the --mark option
""")
parser.add_argument(
"-x", "--exclude", dest='excludes', action='append', default=[],
help="""
like --target, but for specifying exclusions;
for now there no wildcard mechanism is supported here;
also the order in which --target and --exclude options
are mentioned does not matter;
use --dry-run to only check for the list of applicable hosts
""")
# global settings
parser.add_argument(
"-w", "--window", type=int, default=0,
help="""
specify how many connections can run simultaneously;
default is no limit
""")
parser.add_argument(
"-c", "--connect-timeout", dest='timeout',
type=float, default=default_timeout,
help="specify connection timeout, default is {}s"
.format(default_timeout)) # pylint: disable=c0330
# ssh settings
parser.add_argument(
"-l", "--login", default=default_username,
help="remote user name - default is {}".format(default_username))
parser.add_argument(
"-k", "--key", dest='keys',
default=None, action='append', type=str,
help="""
The default is for apssh to locate an ssh-agent
through the SSH_AUTH_SOCK environment variable.
If this cannot be found, or has an empty set of keys,
then the user should specify private key file(s) - additive
""")
parser.add_argument(
"-K", "--ok-if-no-key", default=False, action='store_true',
help="""
When no key can be found, apssh won't even bother
to try and connect. With this option it proceeds
even with no key available.
"""
)
parser.add_argument(
"-g", "--gateway", default=None,
help="""
specify a gateway for 2-hops ssh
- either hostname or username@hostname
""")
# how to store results
# terminal
parser.add_argument(
"-r", "--raw-format", default=False, action='store_true',
help="""
produce raw result, incoming lines are shown as-is without hostname
""")
parser.add_argument(
"-tc", "--time-colon-format", default=False, action='store_true',
help="equivalent to --format '@time@:@host@:@line@")
parser.add_argument(
"-f", "--format", default=None, action='store',
help="""specify output format, which may include
* `strftime` formats like e.g. %%H-%%M, and one of the following:
* @user@ for the remote username,
* @host@ for the target hostname,
* @line@ for the actual line output (which contains the actual newline)
* @time@ is a shorthand for %%H-%%M-%%S""")
# filesystem
parser.add_argument(
"-o", "--out-dir", default=None,
help="specify directory where to store results")
parser.add_argument(
"-d", "--date-time", default=None, action='store_true',
help="use date-based directory to store results")
parser.add_argument(
"-m", "--mark", default=False, action='store_true',
help="""
available with the -d and -o options only.
When specified, then for all nodes there will be a file created
in the output subdir, named either
0ok/<hostname> for successful nodes,
or 1failed/<hostname> for the other ones.
This mark file will contain a single line with the returned code,
or 'None' if the node was not reachable at all
""")
# usual stuff
parser.add_argument(
"-n", "--dry-run", default=False, action='store_true',
help="Only show details on selected hostnames")
parser.add_argument(
"-v", "--verbose",
action='store_true', default=False)
parser.add_argument(
"-D", "--debug", action='store_true', default=False)
parser.add_argument(
"-V", "--version",
action='store_true', default=False)
# the commands to run
parser.add_argument(
"commands", nargs=argparse.REMAINDER, type=str,
help="""
command to run remotely.
If the -s or --script option is provided, the first argument
here should denote a (typically script) file **that must exist**
on the local filesystem. This script is then copied over
to the remote system and serves as the command for remote execution
""")
if test_argv:
args = self.parsed_args = parser.parse_args(test_argv)
else:
args = self.parsed_args = parser.parse_args()
# helpers
if args.version:
print("apssh version {}".format(apssh_version))
exit(0)
# manual check for REMAINDER
if not args.commands:
print("You must provide a command to be run remotely")
parser.print_help()
exit(1)
# load keys
self.loaded_private_keys = load_private_keys(
self.parsed_args.keys, args.verbose or args.debug)
if not self.loaded_private_keys and not args.ok_if_no_key:
print("Could not find any usable key - exiting")
exit(1)
# initialize a gateway proxy if --gateway is specified
gateway = None
if args.gateway:
gwuser, gwhost = self.user_host(args.gateway)
gateway = SshProxy(hostname=gwhost, username=gwuser,
keys=self.loaded_private_keys,
formatter=self.get_formatter(),
timeout=self.parsed_args.timeout,
debug=self.parsed_args.debug)
proxies = self.create_proxies(gateway)
if args.verbose:
print_stderr("apssh is working on {} nodes".format(len(proxies)))
window = self.parsed_args.window
# populate scheduler
scheduler = Scheduler(verbose=args.verbose)
if not args.script:
command_class = Run
extra_kwds_args = {}
else:
# try RunScript
command_class = RunScript
extra_kwds_args = {'includes': args.includes}
# but if the filename is not found then use RunString
script = args.commands[0]
if not Path(script).exists():
if args.verbose:
print("Warning: file not found '{}'\n"
"=> Using RunString instead".
format(script))
command_class = RunString
for proxy in proxies:
scheduler.add(
SshJob(node=proxy,
critical=False,
command=command_class(*args.commands,
**extra_kwds_args)))
# pylint: disable=w0106
scheduler.jobs_window = window
if not scheduler.run():
scheduler.debrief()
results = [job.result() for job in scheduler.jobs]
##########
# print on stdout the name of the output directory
# useful mostly with -d :
subdir = self.get_formatter().run_name \
if isinstance(self.get_formatter(), SubdirFormatter) \
else None
if subdir:
print(subdir)
# details on the individual retcods - a bit hacky
if self.parsed_args.debug:
for proxy, result in zip(proxies, results):
print("PROXY {} -> {}".format(proxy.hostname, result))
# marks
names = {0: '0ok', None: '1failed'}
if subdir and self.parsed_args.mark:
# do we need to create the subdirs
need_ok = [s for s in results if s == 0]
if need_ok:
os.makedirs("{}/{}".format(subdir, names[0]), exist_ok=True)
need_fail = [s for s in results if s != 0]
if need_fail:
os.makedirs("{}/{}".format(subdir, names[None]), exist_ok=True)
for proxy, result in zip(proxies, results):
prefix = names[0] if result == 0 else names[None]
mark_path = Path(subdir) / prefix / proxy.hostname
with mark_path.open("w") as mark:
mark.write("{}\n".format(result))
# xxx - when in gateway mode, the gateway proxy never gets disconnected
# which probably is just fine
# return 0 only if all hosts have returned 0
# otherwise, return 1
failures = [r for r in results if r != 0]
overall = 0 if not failures else 1
return overall