ELAB-partsearch/deploy/Git-Auto-Deploy/gitautodeploy/gitautodeploy.py

if __name__ == '__main__':
    import sys
    print("Critical - GAD must be started as a python module, for example using python -m gitautodeploy")
    sys.exit()


class LogInterface(object):
    """Interface that functions as a stdout and stderr handler and directs the
    output to the logging module, which in turn will output to either console,
    file or both."""

    def __init__(self, level=None):
        import logging
        self.level = (level if level else logging.getLogger().info)

    def write(self, msg):
        for line in msg.strip().split("\n"):
            self.level(line)

    def flush(self):
        pass


from .wsserver import WebSocketClientHandlerFactory
from .httpserver import WebhookRequestHandlerFactory


class GitAutoDeploy(object):
    _instance = None
    _http_server = None
    _https_server = None
    _https_server_unwrapped_socket = None
    _config = {}
    _server_status = {}
    _pid = None
    _event_store = None
    _default_stdout = None
    _default_stderr = None
    _startup_event = None
    _ws_clients = []
    _http_port = None

    def __new__(cls, *args, **kwargs):
        """Overload constructor to enable singleton access"""
        if not cls._instance:
            cls._instance = super(GitAutoDeploy, cls).__new__(
                cls, *args, **kwargs)
        return cls._instance

    def __init__(self):
        from .events import EventStore, StartupEvent

        # Setup an event store instance that can keep a global record of events
        self._event_store = EventStore()
        self._event_store.register_observer(self)

        # Create a startup event that can hold status and any error messages
        # from the startup process
        self._startup_event = StartupEvent()
        self._event_store.register_action(self._startup_event)

    def clone_all_repos(self):
        """Iterates over all configured repositories and clones them to their
        configured paths."""
        import os
        import re
        import logging
        from .wrappers import GitWrapper
        logger = logging.getLogger()

        if 'repositories' not in self._config:
            return

        # Iterate over all configured repositories
        for repo_config in self._config['repositories']:

            # Only clone repositories with a configured path
            if 'url' not in repo_config:
                logger.critical("Repository has no configured URL")
                self.exit()
                return

            # Only clone repositories with a configured path
            if 'path' not in repo_config:
                logger.debug("Repository %s will not be cloned (no path configured)" % repo_config['url'])
                continue

            if os.path.isdir(repo_config['path']) and os.path.isdir(repo_config['path']+'/.git'):
                GitWrapper.init(repo_config)
            else:
                GitWrapper.clone(repo_config)

    def ssh_key_scan(self):
        import re
        import logging
        from .wrappers import ProcessWrapper
        logger = logging.getLogger()

        for repository in self._config['repositories']:

            if 'url' not in repository:
                continue

            logger.info("Scanning repository: %s" % repository['url'])
            m = re.match('[^\@]+\@([^\:\/]+)(:(\d+))?', repository['url'])

            if m is not None:
                host = m.group(1)
                port = m.group(3)
                port_arg = '' if port is None else ('-p %s ' % port)
                cmd = 'ssh-keyscan %s%s >> $HOME/.ssh/known_hosts' % (port_arg, host)
                ProcessWrapper().call([cmd], shell=True)

            else:
                logger.error('Could not find regexp match in path: %s' % repository['url'])

    def create_pid_file(self):
        import os

        with open(self._config['pid-file'], 'w') as f:
            f.write(str(os.getpid()))

    def read_pid_file(self):
        with open(self._config['pid-file'], 'r') as f:
            return f.readlines()

    def remove_pid_file(self):
        import os
        import errno
        if 'pid-file' in self._config and self._config['pid-file']:
            try:
                os.remove(self._config['pid-file'])
            except OSError as e:
                # errno.ENOENT = no such file or directory
                if e.errno != errno.ENOENT:
                    raise

    @staticmethod
    def create_daemon():
        import os

        try:
            # Spawn first child. Returns 0 in the child and pid in the parent.
            pid = os.fork()
        except OSError as e:
            raise Exception("%s [%d]" % (e.strerror, e.errno))

        # First child
        if pid == 0:
            os.setsid()

            try:
                # Spawn second child
                pid = os.fork()

            except OSError as e:
                raise Exception("%s [%d]" % (e.strerror, e.errno))

            if pid == 0:
                os.umask(0)
            else:
                # Kill first child
                os._exit(0)
        else:
            # Kill parent of first child
            os._exit(0)

        return 0

    def update(self, *args, **kwargs):
        import json
        data = json.dumps(kwargs).encode('utf-8')
        for client in self._ws_clients:
            client.sendMessage(data)

    def get_log_formatter(self):
        import logging
        return logging.Formatter("%(asctime)s [%(levelname)-5.5s]  %(message)s")

    def setup_console_logger(self):
        import logging

        # Set up logging
        logger = logging.getLogger()

        consoleHandler = logging.StreamHandler()
        consoleHandler.setFormatter(self.get_log_formatter())

        # Check if a stream handler is already present (will be if GAD is started by test script)
        handler_present = False
        for handler in logger.handlers:
            if isinstance(handler, type(consoleHandler)):
                handler_present = True
                break

        if not handler_present:
            logger.addHandler(consoleHandler)

    def setup(self, config):
        """Setup an instance of GAD based on the provided config object."""
        import sys
        import socket
        import os
        import logging
        import base64
        from .lock import Lock
        import getpass

        # This solves https://github.com/olipo186/Git-Auto-Deploy/issues/118
        try:
            from logging import NullHandler
        except ImportError:
            from logging import Handler

            class NullHandler(Handler):
                def emit(self, record):
                    pass

        # Attatch config values to this instance
        self._config = config

        # Set up logging
        logger = logging.getLogger()
        logFormatter = self.get_log_formatter()

        # Enable console output?
        if ('quiet' in self._config and self._config['quiet']) or ('daemon-mode' in self._config and self._config['daemon-mode']):

            # Add a default null handler that suppresses any console output
            logger.addHandler(NullHandler())

        else:

            # Set up console logger if not already present
            self.setup_console_logger()

        # Set logging level
        if 'log-level' in self._config:
            level = logging.getLevelName(self._config['log-level'])
            logger.setLevel(level)

        if 'log-file' in self._config and self._config['log-file']:
            # Translate any ~ in the path into /home/<user>
            fileHandler = logging.FileHandler(self._config['log-file'])
            fileHandler.setFormatter(logFormatter)
            logger.addHandler(fileHandler)

        # Display a warning when trying to run as root
        if not self._config['allow-root-user'] and getpass.getuser() == 'root':
            logger.critical("Refusing to start as root. This application shouldn't run as a privileged used. Please run it as a different user. To disregard this warning and start anyway, set the config option \"allow-root-user\" to true, or use the command line argument --allow-root-user")
            sys.exit()

        if 'ssh-keyscan' in self._config and self._config['ssh-keyscan']:
            self._startup_event.log_info('Scanning repository hosts for ssh keys...')
            self.ssh_key_scan()

        # Clone all repos once initially
        self.clone_all_repos()

        # Set default stdout and stderr to our logging interface (that writes
        # to file and console depending on user preference)
        if 'intercept-stdout' in self._config and self._config['intercept-stdout']:
            self._default_stdout = sys.stdout
            self._default_stderr = sys.stderr
            sys.stdout = LogInterface(logger.info)
            sys.stderr = LogInterface(logger.error)

        if 'daemon-mode' in self._config and self._config['daemon-mode']:
            self._startup_event.log_info('Starting Git Auto Deploy in daemon mode')
            GitAutoDeploy.create_daemon()

        self._pid = os.getpid()
        self.create_pid_file()

        # Generate auth key to protect the web socket server
        self._server_status['auth-key'] = base64.b64encode(os.urandom(32))

        # Clear any existing lock files, with no regard to possible ongoing processes
        for repo_config in self._config['repositories']:

            # Do we have a physical repository?
            if 'path' in repo_config:
                Lock(os.path.join(repo_config['path'], 'status_running')).clear()
                Lock(os.path.join(repo_config['path'], 'status_waiting')).clear()

        #if 'daemon-mode' not in self._config or not self._config['daemon-mode']:
        #    self._startup_event.log_info('Git Auto Deploy started')

    def serve_http(self, serve_forever=True):
        """Starts a HTTP server that listens for webhook requests and serves the web ui."""
        import sys
        import socket
        import os
        from .events import SystemEvent

        try:
            from BaseHTTPServer import HTTPServer
        except ImportError as e:
            from http.server import HTTPServer

        if not self._config['http-enabled']:
            return

        # Setup
        try:

            # Create web hook request handler class
            WebhookRequestHandler = WebhookRequestHandlerFactory(self._config, self._event_store, self._server_status, is_https=False)

            # Create HTTP server
            self._http_server = HTTPServer((self._config['http-host'],
                                       self._config['http-port']),
                                      WebhookRequestHandler)

            # Setup SSL for HTTP server
            sa = self._http_server.socket.getsockname()
            self._http_port = sa[1]
            self._server_status['http-uri'] = "http://%s:%s" % (self._config['http-host'], sa[1])
            self._startup_event.log_info("Listening for connections on %s" % self._server_status['http-uri'])
            self._startup_event.http_address = sa[0]
            self._startup_event.http_port = sa[1]
            self._startup_event.set_http_started(True)

        except socket.error as e:
            self._startup_event.log_critical("Unable to start HTTP server: %s" % e)
            return

        if not serve_forever:
            return

        # Run forever
        try:
            self._http_server.serve_forever()

        except socket.error as e:
            event = SystemEvent()
            self._event_store.register_action(event)
            event.log_critical("Error on socket: %s" % e)
            sys.exit(1)

        except KeyboardInterrupt as e:
            event = SystemEvent()
            self._event_store.register_action(event)
            event.log_info('Requested close by keyboard interrupt signal')
            self.stop()
            self.exit()

        event = SystemEvent()
        self._event_store.register_action(event)
        event.log_info('HTTP server did quit')

    def serve_https(self):
        """Starts a HTTPS server that listens for webhook requests and serves the web ui."""
        import sys
        import socket
        import os
        import ssl
        from .events import SystemEvent

        try:
            from BaseHTTPServer import HTTPServer
        except ImportError as e:
            from http.server import HTTPServer

        if not self._config['https-enabled']:
            return

        if not os.path.isfile(self._config['ssl-cert']):
            self._startup_event.log_critical("Unable to activate SSL: File does not exist: %s" % self._config['ssl-cert'])
            return

        # Setup
        try:

            # Create web hook request handler class
            WebhookRequestHandler = WebhookRequestHandlerFactory(self._config, self._event_store, self._server_status, is_https=True)

            # Create HTTP server
            self._https_server = HTTPServer((self._config['https-host'],
                                       self._config['https-port']),
                                      WebhookRequestHandler)

            # Setup SSL for HTTP server
            self._https_server_unwrapped_socket = self._https_server.socket
            self._https_server.socket = ssl.wrap_socket(self._https_server.socket,
                                                keyfile=self._config['ssl-key'],
                                                certfile=self._config['ssl-cert'],
                                                server_side=True)

            sa = self._https_server.socket.getsockname()
            self._http_port = sa[1]
            self._server_status['https-uri'] = "https://%s:%s" % (self._config['https-host'], sa[1])

            self._startup_event.log_info("Listening for connections on %s" % self._server_status['https-uri'])
            self._startup_event.http_address = sa[0]
            self._startup_event.http_port = sa[1]
            self._startup_event.set_http_started(True)

        except socket.error as e:
            self._startup_event.log_critical("Unable to start HTTPS server: %s" % e)
            return

        # Run forever
        try:
            self._https_server.serve_forever()

        except socket.error as e:
            event = SystemEvent()
            self._event_store.register_action(event)
            event.log_critical("Error on socket: %s" % e)
            sys.exit(1)

        except KeyboardInterrupt as e:
            event = SystemEvent()
            self._event_store.register_action(event)
            event.log_info('Requested close by keyboard interrupt signal')
            self.stop()
            self.exit()

        event = SystemEvent()
        self._event_store.register_action(event)
        event.log_info('HTTPS server did quit')

    def serve_wss(self):
        """Start a web socket server over SSL, used by the web UI to get notifications about updates."""
        import os
        from .events import SystemEvent

        # Start a web socket server if the web UI is enabled
        if not self._config['web-ui-enabled']:
            return

        if not self._config['wss-enabled']:
            return

        if not os.path.isfile(self._config['ssl-cert']):
            self._startup_event.log_critical("Unable to activate SSL: File does not exist: %s" % self._config['ssl-cert'])
            return

        try:
            import os
            from autobahn.websocket import WebSocketServerProtocol, WebSocketServerFactory
            from twisted.internet import reactor, ssl
            from twisted.internet.error import BindError

            # Create a WebSocketClientHandler instance
            WebSocketClientHandler = WebSocketClientHandlerFactory(self._config, self._ws_clients, self._event_store, self._server_status)

            uri = u"ws://%s:%s" % (self._config['wss-host'], self._config['wss-port'])
            factory = WebSocketServerFactory(uri)
            factory.protocol = WebSocketClientHandler
            # factory.setProtocolOptions(maxConnections=2)

            # note to self: if using putChild, the child must be bytes...
            if self._config['ssl-key'] and self._config['ssl-cert']:
                contextFactory = ssl.DefaultOpenSSLContextFactory(privateKeyFileName=self._config['ssl-key'], certificateFileName=self._config['ssl-cert'])
            else:
                contextFactory = ssl.DefaultOpenSSLContextFactory(privateKeyFileName=self._config['ssl-cert'], certificateFileName=self._config['ssl-cert'])


            self._ws_server_port = reactor.listenSSL(self._config['wss-port'], factory, contextFactory)
            # self._ws_server_port = reactor.listenTCP(self._config['wss-port'], factory)

            self._server_status['wss-uri'] = "wss://%s:%s" % (self._config['wss-host'], self._config['wss-port'])

            self._startup_event.log_info("Listening for connections on %s" % self._server_status['wss-uri'])
            self._startup_event.ws_address = self._config['wss-host']
            self._startup_event.ws_port = self._config['wss-port']
            self._startup_event.set_ws_started(True)

            # Serve forever (until reactor.stop())
            reactor.run(installSignalHandlers=False)

        except BindError as e:
            self._startup_event.log_critical("Unable to start web socket server: %s" % e)

        except ImportError:
            self._startup_event.log_error("Unable to start web socket server due to missing dependency.")

        event = SystemEvent()
        self._event_store.register_action(event)
        event.log_info('WSS server did quit')

    def serve_forever(self):
        """Start HTTP and web socket servers."""
        import sys
        import socket
        import logging
        import os
        from .events import SystemEvent
        import threading

        try:
            from autobahn.websocket import WebSocketServerProtocol, WebSocketServerFactory
            from twisted.internet import reactor

            # Given that the nessecary dependencies are present, notify the
            # event that we expect the web socket server to be started
            self._startup_event.ws_started = False
        except ImportError:
            pass

        # Notify the event that we expect the http server to be started
        self._startup_event.http_started = False

        # Add script dir to sys path, allowing us to import sub modules even after changing cwd
        sys.path.insert(1, os.path.dirname(os.path.realpath(__file__)))

        # Set CWD to public www folder. This makes the http server serve files from the wwwroot directory.
        wwwroot = os.path.join(os.path.dirname(os.path.realpath(__file__)), "wwwroot")
        os.chdir(wwwroot)

        threads = [
            # HTTP server
            threading.Thread(target=self.serve_http),

            # HTTPS server
            threading.Thread(target=self.serve_https),

            # Web socket SSL server
            threading.Thread(target=self.serve_wss)
        ]

        # Start all threads
        for thread in threads:
            thread.start()

        # Wait for each thread to finish
        for thread in threads:

            # Wait for thread to finish without blocking main thread
            while thread.is_alive():
                thread.join(5)

    def signal_handler(self, signum, frame):
        from .events import SystemEvent
        self.stop()

        event = SystemEvent()
        self._event_store.register_action(event)

        # Reload configuration on SIGHUP events (conventional for daemon processes)
        if signum == 1:
            self.setup(self._config)
            self.serve_forever()
            return

        # Keyboard interrupt signal
        elif signum == 2:
            event.log_info('Recieved keyboard interrupt signal (%s) from the OS, shutting down.' % signum)

        else:
            event.log_info('Recieved signal (%s) from the OS, shutting down.' % signum)

        self.exit()

    def stop(self):
        """Stop all running TCP servers (HTTP and web socket servers)"""

        # Stop HTTP server if running
        if self._http_server is not None:

            # Shut down the underlying TCP server
            self._http_server.shutdown()

            # Close the socket
            self._http_server.socket.close()

        # Stop HTTPS server if running
        if self._https_server is not None:

            # Shut down the underlying TCP server
            self._https_server.shutdown()

            # Close the socket
            self._https_server.socket.close()

        if self._https_server_unwrapped_socket is not None:

            self._https_server_unwrapped_socket.close()

        # Stop web socket server if running
        try:
            from twisted.internet import reactor
            reactor.callFromThread(reactor.stop)
        except ImportError:
            pass

    def exit(self):
        import sys
        import logging
        logger = logging.getLogger()
        logger.info('Goodbye')

        # Delete PID file
        self.remove_pid_file()

        # Restore stdin and stdout
        if 'intercept-stdout' in self._config and self._config['intercept-stdout']:
            sys.stdout = self._default_stdout
            sys.stderr = self._default_stderr


def main():
    import signal
    from gitautodeploy import GitAutoDeploy
    from cli.config import get_config_defaults, get_config_from_environment
    from cli.config import get_config_from_argv, find_config_file
    from cli.config import get_config_from_file, get_repo_config_from_environment
    from cli.config import init_config, get_config_file_path, rename_legacy_attribute_names
    from cli.config import ConfigFileNotFoundException, ConfigFileInvalidException
    import logging
    import sys
    import os

    logger = logging.getLogger()

    app = GitAutoDeploy()

    if hasattr(signal, 'SIGHUP'):
        signal.signal(signal.SIGHUP, app.signal_handler)
    if hasattr(signal, 'SIGINT'):
        signal.signal(signal.SIGINT, app.signal_handler)
    if hasattr(signal, 'SIGABRT'):
        signal.signal(signal.SIGABRT, app.signal_handler)
    if hasattr(signal, 'SIGPIPE') and hasattr(signal, 'SIG_IGN'):
        signal.signal(signal.SIGPIPE, signal.SIG_IGN)

    # Get default config values
    config = get_config_defaults()

    # Get config values from environment variables and commadn line arguments
    environment_config = get_config_from_environment()
    argv_config = get_config_from_argv(sys.argv[1:])

    # Merge config values from environment variables
    config.update(environment_config)

    search_target = os.path.dirname(os.path.realpath(__file__))
    config_file_path = get_config_file_path(environment_config, argv_config, search_target)

    # Config file path provided or found?
    if config_file_path:

        try:
            file_config = get_config_from_file(config_file_path)
        except ConfigFileNotFoundException as e:
            app.setup_console_logger()
            logger.critical("No config file not found at '%s'" % e)
            return
        except ConfigFileInvalidException as e:
            app.setup_console_logger()
            logger.critical("Unable to read config file due to invalid JSON format in '%s'" % e)
            return

        # Merge config values from config file (overrides environment variables)
        config.update(file_config)

    # Merge config value from command line (overrides environment variables and config file)
    config.update(argv_config)

    # Rename legacy config option names
    config = rename_legacy_attribute_names(config)

    # Extend config data with any repository defined by environment variables
    repo_config = get_repo_config_from_environment()

    if repo_config:

        if not 'repositories' in config:
            config['repositories'] = []

        config['repositories'].append(repo_config)

    # Initialize config by expanding with missing values
    init_config(config)

    app.setup(config)
    app.serve_forever()