1#@!#!123s

��JFIF�� 1#@!#!123s
: / opt / sharedrads / oldrads /
Filename : autosuspend.py
back
#!/opt/imh-python/bin/python3
"""Automatic resource overage suspension script"""
import datetime
import socket
import sys
import re
import sh
import time
import yaml
import os
import configparser
import pp_api
import logging
import pwd
from collections import defaultdict
from multiprocessing import cpu_count
from functools import partial
from rads.shared import (
    is_suspended,
    is_cpanel_user,
    get_secure_username,
    SYS_USERS,
)


class Autosuspend:
    """
    Gathers current and historic user cp usage data and determines
    whether or not to enact an account suspension, send a warning
    or pass over system users
    """

    config_files = [
        '/opt/sharedrads/etc/autosuspend.cfg',
        '/opt/sharedrads/etc/autosuspend.cfg.local',
    ]

    brand = ('imh', 'hub')['hub' in socket.gethostname()]

    def __init__(self):
        """
        Initializes an instance of Autosuspend, including a logging
        object, parsed config from Autosuspend.config_files and
        various information on system users
        """
        self.config = configparser.ConfigParser(allow_no_value=False)
        self.config.read(self.config_files)

        logging.basicConfig(
            level=logging.INFO,
            format=f'%(asctime)s {sys.argv[0]}: %(message)s',
            datefmt='%Y-%m-%d:%H:%M:%S %Z',
            filename=self.suspension_log,
        )
        self.logger = logging.getLogger('suspension_logger')

        self.priors = prior_events(
            data_file=self.data_file,
            log=self.suspension_log,
            log_offset=self.suspension_log_offset,
        )

        self.suspensions_enabled = self.config.getboolean(
            'suspensions',
            'enabled',
        )
        self.warnings_enabled = self.config.getboolean(
            'warnings',
            'enabled',
        )

        self.freepass_enabled = self.config.getboolean(
            'freepass',
            'enabled',
        )

        self.actions = {
            'suspension': enact_suspension,
            'warning': send_warning,
            'freepass': give_free_pass,
        }

        self.server_overloaded = server_overloaded()

        _users = top_users(
            interval_file=self.sa_interval_file,
            max_age=self.sa_interval_file_max_age,
        )

        self.users = {
            name: User(
                name=name,
                delta=delta,
                suspensions=self.priors.get(name, {}).get('suspensions', []),
                warnings=self.priors.get(name, {}).get('warnings', []),
                freepasses=self.priors.get(name, {}).get('freepasses', []),
            )
            for name, delta in _users
            if not is_suspended(name)
        }

    def __repr__(self):
        """
        Returns a representation of an Autosuspend object
        """
        repr_data = [
            'brand',
            'disruptive_action_interval',
            'server_load_factor',
            'server_overloaded',
            'suspensions_enabled',
            'warnings_enabled',
            'freepass_enabled',
        ]
        repr_str = '<Autosuspend {}>'.format(
            ' '.join([f'{i}:{getattr(self, i)}' for i in repr_data])
        )
        return repr_str

    def suspension_critera_met(self, user):
        """
        Tests a User object to see if it meets suspension criteria
        """
        if not user.warned_within(self.disruptive_action_interval):
            self.logger.debug(
                f'{user.name} not warned within {self.disruptive_action_interval}, not suspending'
            )
            return False

        # double check this logic - if user was suspended longer ago than action_interval they should be elligible
        if not user.suspended_longer_ago(self.disruptive_action_interval):
            self.logger.debug(
                f'{user.name} not suspended within {self.disruptive_action_interval}, not suspending'
            )
            return False

        if user.num_warnings <= self.warning_count:
            self.logger.debug(
                f'Not suspended; only {user.num_warnings} warnings, need {self.warning_count}'
            )
            return False

        if float(user.delta) >= float(self.suspensions['max_delta']):
            return True

        return False

    def warning_critera_met(self, user):
        """
        Tests a User object to see if it meets warning criteria
        """

        if user.warned_within(self.disruptive_action_interval):
            self.logger.debug(
                f'{user.name} warned within {self.disruptive_action_interval}, not warning'
            )
            return False

        if float(user.delta) >= float(self.warnings['max_delta']):
            return True
        else:
            self.logger.debug(
                '{} has not consumed more than {}cp in the last {}'.format(
                    user.name,
                    self.warnings['max_delta'],
                    self.disruptive_action_interval,
                )
            )

        return False

    def freepass_criteria_met(self, user):
        """
        Tests a user to see if it meets freepass criteria
        """
        self.logger.debug(f'Testing {user.name} for freepass...')
        if float(user.delta) >= float(self.freepass['max_delta']):
            self.logger.debug(
                f'{user.name} has a delta of {user.delta}, which is above the threshold.'
            )
            if not user.given_freepass_within(self.time_between_free_passes):
                self.logger.debug(
                    f'{user.name} was not given a free pass within {self.time_between_free_passes} so they get one'
                )
                return True
            else:
                self.logger.debug(
                    f'{user.name} got a free pass within the last {self.time_between_free_passes} days, not sending another'
                )

        return False

    def run(self):
        """
        Loops through Autosuspend.users, calling Autosuspend.test for each

        """
        self.logger.info(f'Autosuspend run starting {repr(self)}')
        if not self.users:
            return

        for user in self.users.values():
            action = self.test(user)
            action_func = self.actions.get(
                action,
                lambda *x, **y: None,
            )
            wrapper = partial(
                action_func, email_template=getattr(self, f'{action}_template')
            )

            wrapper(user=user.name, comment=user.note)
        self.logger.info('Autosuspend run complete')

    def test(self, user):
        """
        Determines what action, if any to take against an individual
        User object
        """
        user.suspend = self.suspension_critera_met(user)
        user.warn = self.warning_critera_met(user)
        user.freepass = self.freepass_criteria_met(user)

        if user.suspend and self.suspensions_enabled and self.server_overloaded:
            user.note = (
                'AUTO SUSPENSION: Consumed {:.2f}cp within '
                'the last measured interval.'.format(user.delta)
            )
            self.logger.debug(f'Suspending {user}')
            self.logger.info(
                f'{user.delta} [AUTO_SUSPENSION] ra - root "{user.note}"'
            )
            return 'suspension'

        elif user.freepass and self.freepass_enabled:
            user.note = (
                'AUTO RA FREEPASS: Consumed {:.2f}cp within '
                'the last measured interval.'.format(user.delta)
            )
            self.logger.debug(f'Freepassing {user}')
            self.logger.info(f'{user.name} [FREEPASS] ra - root "{user.note}"')
            return 'freepass'

        elif user.warn and self.warnings_enabled:
            user.note = (
                'AUTO RA WARNING: Consumed {:.2f}cp within '
                'the last measured interval.'.format(user.delta)
            )
            self.logger.debug(f'Warning {user}')
            self.logger.info(f'{user.name} [WARNING] ra - root "{user.note}"')
            return 'warning'
        else:
            self.logger.debug(f'Skipping {user}')
            return 'skip'

    def __getattr__(self, item):
        """
        Returns items as strings from settings and brand-specific settings
        sections or entire config sections as a dict

        e.g. <Autosuspend instance>.suspension_log;
        <Autosuspend instance>.settings['suspension_log']
        """
        if item in self.config.sections():
            return dict(self.config.items(item))
        # See if a given key is present in the settings section
        for section in (f'settings_{self.brand}', 'settings'):
            if self.config.has_option(section, item):
                return self.config.get(section, item)


class User:
    """
    Instantiated to represent a system user.
    """

    def __init__(self, **args):
        """
        Initializes the User object
        """
        self.data_dict = args
        self.warn = False
        self.suspend = False
        self.freepass = False
        self.num_suspensions = len(args['suspensions'])
        self.num_warnings = len(args['warnings'])
        self.num_freepasses = len(args['freepasses'])

    def __getattr__(self, item):
        """
        Returns an item from self.data_dict or None in the event of a KeyError
        """
        try:
            return self.data_dict[item]
        except KeyError:
            pass

    def __repr__(self):
        """
        Returns a useful representation of a User object
        """
        repr_data = [
            'name',
            'delta',
            'last_suspension',
            'last_warning',
            'last_freepass',
            'num_suspensions',
            'num_warnings',
            'num_freepasses',
            'suspend',
            'warn',
            'freepass',
            'note',
        ]

        repr_str = '<User {}>'.format(
            ' '.join([f'{i}:{getattr(self, i)}' for i in repr_data])
        )
        return repr_str

    def warned_within(self, delta):
        """
        Returns True if the user's last warning was sent within the current
        time - delta, False otherwise
        """
        if not isinstance(delta, datetime.timedelta):
            delta = str_to_timedelta(delta)
        try:
            return datetime.datetime.now() < (self.last_warning + delta)
        except TypeError:
            return False

    def suspended_longer_ago(self, delta):
        """
        Returns True if the user's last suspension was longer ago
        than the current time - delta, False otherwise
        """
        if not isinstance(delta, datetime.timedelta):
            delta = str_to_timedelta(delta)
        try:
            return datetime.datetime.now() > (self.last_suspension + delta)
        except TypeError:
            return True

    def given_freepass_within(self, delta):
        """
        In the case self.last_freepass is None, we return false.
        """
        if not self.last_freepass:
            return False
        if not isinstance(delta, datetime.timedelta):
            delta = str_to_timedelta(delta)
        try:
            return datetime.datetime.now() < (self.last_freepass + delta)
        except TypeError:
            return True

    @property
    def last_suspension(self):
        """
        returns a datetime object which represents the last time
        the user was suspended or None
        """
        return self._last_suspension

    @last_suspension.getter
    def last_suspension(self):
        """
        returns a datetime object which represents the last time
        the user was suspended or None
        """
        return self._nth_date('suspensions', -1)

    @property
    def last_warning(self):
        """
        returns a datetime object which represents the last time
        the user was warned or None
        """
        return self._last_warning

    @last_warning.getter
    def last_warning(self):
        """
        returns a datetime object which represents the last time
        the user was warned or None
        """
        return self._nth_date('warnings', -1)

    @property
    def last_freepass(self):
        return self._last_freepass

    @last_freepass.getter
    def last_freepass(self):
        return self._nth_date('freepasses', -1)

    def _nth_date(self, attr, index):
        """
        Return a datetime object representation of a date from
        suspension or warning lists
        """
        items = getattr(self, attr)
        try:
            return datetime.datetime.fromtimestamp(
                sorted(map(float, items))[index]
            )
        except (TypeError, IndexError):
            pass


def str_to_timedelta(time_str):
    """
    Munges strings into timedelta objects
    """
    match = re.search(
        r"""(:?
        (:?(?P<hours>\d+)[Hh])?
        (:?(?P<minutes>\d+)[Mm])?
        (:?(?P<days>\d+)[Dd])?
        (:?(?P<seconds>\d+)[Ss])?
        )+""",
        ''.join(time_str.split()),
        re.VERBOSE,
    )
    groups = {k: float(v) for k, v in match.groupdict().items() if v}
    return datetime.timedelta(**groups)


def server_overloaded(factor=1.5):
    """
    Determines whether or not the sever is unduly stressed by comparing the
    15-minute load average and the product of number of cores and 'factor'.
    """
    return (cpu_count() * factor) <= os.getloadavg()[-1]


def try_open_yaml(yaml_path):
    """Try to read a yaml file. If impossible, return an empty dict"""
    try:
        data = yaml.load(file(yaml_path, 'r'))
    except (OSError, yaml.error.MarkedYAMLError):
        return {}
    if not isinstance(data, dict):
        return {}
    return data


def get_log_data(logfile, offsetfile, ignore_offset=False):
    """
    Reads and offset from the offset file, returns data from the offset to
    the end of the file
    """

    # try to read the offset from the offset file
    try:
        with open(offsetfile) as offset_fh:
            offset = int(offset_fh.readline())
    # Set offset to 0 if the offset can't be converted to an integer or the
    # file is missing
    except (OSError, ValueError):
        offset = 0

    if ignore_offset:
        offset = 0

    try:
        with open(logfile) as logfile_fh:
            logfile_fh.seek(0, 2)
            logfile_length = logfile_fh.tell()
            if offset > logfile_length:
                logfile_fh.seek(0)
            else:
                logfile_fh.seek(offset)
            output = logfile_fh.readlines()
            newoffset = logfile_fh.tell()
    # If the file can't be opened return an empty string
    # and set newoffset to 0
    except OSError:
        output = ""
        newoffset = 0

    # Write the new offset to the offset file
    with open(offsetfile, 'w') as offset_fh:
        offset_fh.write(str(newoffset))
    return output


def prior_events(log=None, log_offset=None, data_file=None):
    '''Returns a dict that contains account suspension times'''
    suspension_re = re.compile(
        r"""(?P<time>\d{4}-\d{2}-\d{2}:\d{2}:\d{2}:\d{2})
        \s\w+\s+[\w/\.-]+:\s+(?P<user>\w+)\s+\[
        (:?
        (?P<suspensions>(:?AUTO_)?SUSPENSION)|
        (?P<warnings>WARNING)|
        (?P<freepasses>FREEPASS)
        )
        \]\s+ra""",
        re.VERBOSE,
    )
    priors = defaultdict(
        lambda: {'suspensions': [], 'warnings': [], 'freepasses': []}
    )

    # Get prior suspensions
    if data_file is not None:
        priors.update(try_open_yaml(data_file))

    # If past_suspensions wasn't populated with any data, skip using the offset
    # when reading log data
    if len(priors) < 1:
        skip_offset = True
    else:
        skip_offset = False

    # Lolast_freepassop through new lines from the suspension log and add to the suspension
    # data dict
    for line in get_log_data(log, log_offset, skip_offset):
        match = suspension_re.search(line)
        if match:
            user = match.group('user')
            event_time = int(
                time.mktime(
                    time.strptime(match.group('time'), "%Y-%m-%d:%H:%M:%S")
                )
            )
            for event in ('suspensions', 'warnings', 'freepasses'):
                if match.group(event):
                    try:
                        priors[user][event].append(event_time)
                    except (KeyError, AttributeError):
                        priors[user][event] = [event_time]

    yaml.dump(dict(list(priors.items())), file(data_file, 'w+'))

    return priors


def prepare_str_content(user):
    """Runs various commands, returns string output"""
    output = []

    rads_cmds = (
        ('/opt/sharedrads/check_user', dict(plaintext=True)),
        (
            '/opt/sharedrads/nlp',
            dict(w=80, p=True, _err_to_out=True, today=True),
        ),
        ('/opt/sharedrads/recent-cp', dict(b=True)),
    )

    for script, kwargs in rads_cmds:
        # these RADS scripts all accept a user name as the first
        # positional argument so it can just be baked in
        try:
            cmd = sh.Command(script).bake(user)
        except sh.CommandNotFound:
            continue

        try:
            result = cmd(**kwargs)
        except sh.ErrorReturnCode:
            continue

        output.append(
            (
                f'>>> {result.ran}',
                result.stdout,
            )
        )

    output.append(
        (
            '>>> Running processes prior to suspension',
            sh.awk(sh.ps('auwwx', _piped=True), "$1 ~ /%s|USER/" % user).stdout,
        )
    )

    output = "\n\n".join(['\n'.join(items) for items in output])
    return output


def save_str(user, content):
    """
    Saves STR data in the user's '.imh' folder
    """
    imhdir = os.path.join(pwd.getpwnam(user).pw_dir, '.imh')
    if not os.path.isdir(imhdir):
        try:
            os.mkdir(imhdir)
        except OSError:
            return

    date_string = datetime.datetime.strftime(
        datetime.datetime.now(), '%Y-%m-%d_%H:%M:%S'
    )

    strfile = os.path.join(imhdir, f'str_{date_string}')

    with open(strfile, 'w') as str_fh:
        str_fh.write(content)


def send_str(user, content):
    """Sends an STR to the STR queue"""
    strmailer_cmd = sh.Command('/opt/sharedrads/mailers/strmailer-suspend.py')
    strmailer_cmd(user, socket.gethostname().split('.')[0], _in=content)


def enact_suspension(user, comment, email_template):
    """Suspends the user, notes the account and sends an STR"""
    ppa = pp_api.PowerPanel()

    ppa.call('notification.send', template=email_template, cpanelUser=user)
    ppa.call(
        'hosting.insert-note',
        admin_user='auto',
        user=user,
        note='AUTO SUSPENSION: %s' % comment,
        flagged=True,
        type='Suspension',
    )

    message = 'RADS suspended {} for resource abuse.\n\n{}\n\n' '{}'.format(
        user, comment, prepare_str_content(user)
    )

    send_str(user, message)
    save_str(user, message)

    suspend_cmd = sh.Command('/opt/sharedrads/suspend_user')
    suspend_cmd(
        user, invoked_by=sys.argv[0], ra=True, c=comment, _tty_out=False
    )


def send_warning(user, comment, email_template):
    """Sends a warning to the user, notes the customer's account"""
    ppa = pp_api.PowerPanel()
    cmd = sh.Command(
        '/opt/sharedrads/python/send_customer_str/send_customer_str'
    )
    run = cmd('-u', user)
    if run.exit_code != 0:
        send_str(
            user, "Could not generate STR for customer with send_customer_str"
        )
        ppa.call('notification.send', template=email_template, cpanelUser=user)

    ppa.call(
        'hosting.insert-note',
        admin_user='auto',
        user=user,
        note='AUTO RA NOTICE: %s' % comment,
        flagged=True,
        type='Resource Management',
    )

    save_str(user, prepare_str_content(user))


def give_free_pass(user, comment, email_template):
    save_str(user, prepare_str_content(user))


def top_users(interval_file, max_age, num_users=None):
    """
    Return sorted top users in the last time interval. If num_users specified,
    only return that number of users. Otherwise, return all of them. This
    function also handles saving the current sa data for the next run
    """
    this_sa = get_sa_dict()
    user_deltas = []
    if check_interval_file_age(interval_file, max_age):
        last_sa = try_open_yaml(interval_file)

        for user in this_sa.keys():
            try:
                delta = float(this_sa[user]) - float(last_sa[user])
            except (KeyError, ValueError):
                continue
            user_deltas.append((user, delta))

        # sort by usage, descending
        user_deltas.sort(key=lambda tup: tup[1], reverse=True)

    # save this run's sa data for the next run
    with open(interval_file, 'w') as data_file:
        yaml.dump(
            this_sa,
            data_file,
        )
    return user_deltas[:num_users]


def get_sa_dict():
    """get info from 'sa -m' in the form of a dict"""
    sec_usr = get_secure_username()
    skip_user_re = re.compile(
        r'(?:{}{})\b'.format(
            '' if sec_usr is None else '%s|' % sec_usr, '|'.join(SYS_USERS)
        )
    )
    sa_re = re.compile(
        r'^(?P<user>\w+) +[0-9]+ +[0-9]+\.[0-9]{2}'
        r're +(?P<cp>[0-9]+\.[0-9]{2})cp '
    )
    sa_output = [
        x for x in sh.sa(m=True).splitlines() if not skip_user_re.match(x)
    ]
    sa_dict = {}
    for line in sa_output:
        match = sa_re.search(line)
        if match is not None:
            user, cp_total = match.group('user'), match.group('cp')
            if not is_cpanel_user(user):
                continue
            sa_dict[user] = float(cp_total)
    return sa_dict


def check_interval_file_age(interval_file_path, max_age):
    """
    Check how old the sa_interval_file is. Return true if the age is what
    is expected from autosuspend.cfg. If false, the file's contents cannot
    be used to decide whether users should be suspended
    """
    if not isinstance(max_age, datetime.timedelta):
        max_age = str_to_timedelta(max_age)
    try:
        mtime = datetime.datetime.fromtimestamp(
            os.path.getmtime(interval_file_path)
        )
    except OSError:  # file not found
        return False
    return (datetime.datetime.now() - mtime) < max_age


if __name__ == '__main__':
    Autosuspend().run()