services: make services deployable from single command

services/audiobookshelf/init.sh

@@ -1,10 +0,0 @@
-#! /usr/bin/bash
-
-source ./.env
-set -xe
-
-mkdir -p "${DOCKER_PARENT_PATH}"
-mkdir -p "${SVC_PATH}"
-mkdir -p "${SVC_PATH}/config"
-mkdir -p "${SVC_PATH}/metadata"
-

services/ippoolmanager-sql-server /.env

@@ -1,2 +0,0 @@
-MYSQL_ROOT_PASSWORD="2F&B7yUoaE9F"
-MYSQL_USER_PASSWORD="ippoolmanager"

services/ippoolmanager-sql-server/.env

@@ -0,0 +1,4 @@
+MYSQL_ROOT_PASSWORD="2F&B7yUoaE9F"
+MYSQL_USER_PASSWORD="ippoolmanager"
+USER_PASSWORD="wxCD2JKXWbjCyBd504Q43Uv2W6gg80Y671"
+ROOT_PASSWORD="FMB2u7Lhgt282rYPW914J1ivg1bn5YPt0X"

services/ippoolmanager-sql-server/docker-compose.yml

@@ -1,4 +1,3 @@
-version: '3.3'
 services:
   db:
     image: mysql:latest
@@ -9,7 +8,7 @@ services:
       MYSQL_PASSWORD: '${USER_PASSWORD}'
       MYSQL_ROOT_PASSWORD: '${ROOT_PASSWORD}'
     ports:
-      - '192.168.100.57:3306:3306'
+      - '3306:3306'
     expose:
       - '3306'
     volumes:

services/jenkins/ci/jobs/modules/Utils.groovy

@@ -0,0 +1,43 @@
+#!/usr/bin/env groovy
+
+package modules
+
+import javaposse.jobdsl.dsl.Job
+
+public class Utils {
+    static Job createJob(
+        String jobName,
+        String jobDescription,
+        jobDslCtx
+    ) {
+        Job job = jobDslCtx.pipelineJob(jobName)
+        job.with {
+            description(jobDescription)
+        }
+        return job
+    }
+
+    static void addGiteaRepository(
+        Job job,
+        String giteaRepositoryURL,
+        String jenkinsFilePath
+    ) {
+        job.with {
+            definition {
+                cpsScm {
+                    lightweight(false)
+                    scm {
+                        git {
+                            branch('main')
+                            remote {
+                                url(giteaRepositoryURL)
+                                credentials('77c771b9-20f4-426d-8f1e-ed901afe9eb9')
+                            }
+                        }
+                    }
+                    scriptPath(jenkinsFilePath)
+                }
+            }
+        }
+    }
+}

services/jenkins/ci/jobs/seed.groovy

@@ -0,0 +1,18 @@
+#!/usr/bin/env groovy
+import javaposse.jobdsl.dsl.Job
+import modules.Utils
+
+void createSeed() {
+    String jobName = 'seed'
+    String jobDescription = 'This job creates all other jobs'
+    String jenkinsFile = 'services/jenkins/ci/pipelines/seed.groovy'
+    String repoUrl = 'https://git.tylencloud.com/tylen/andromeda-setup'
+
+    Job job = Utils.createJob(jobName, jobDescription, this)
+    Utils.addGiteaRepository(
+        job,
+        repoUrl,
+        jenkinsFile
+    )
+    
+}

services/services.env

@@ -1 +0,0 @@
-# Environment variables generated from JSON data

services/services.yaml

@@ -0,0 +1,22 @@
+defaultServiceValues: &defaultServiceValues
+  composeFile: "docker-compose.yml"
+  envFile: ".env"
+
+vm-tools-100-65: &vm-tools-100-65
+  ip: "192.168.100.65"
+  user: vm-user
+
+vm-media-100-55: &vm-media-100-55
+  ip: "192.168.100.55"
+  user: vm-user
+
+services:
+  - name: "audiobookshelf"
+    host:
+      <<: *vm-media-100-55
+    <<: *defaultServiceValues
+  
+  - name: "ippoolmanager-sql-server"
+    host:
+      <<: *vm-tools-100-65
+    <<: *defaultServiceValues

services/tools/cli/args_utils.py

@@ -1,5 +1,6 @@
 from argparse import ArgumentTypeError
 import os
+import subprocess
 
 def get_full_path(relative_path):
     if not os.path.isabs(relative_path):
@@ -15,4 +16,23 @@ def dir_arg_type(path):
     if os.path.isdir(full_path):
         return full_path
     else:
-        raise ArgumentTypeError(f"{path} nor {full_path} are not a valid path")
+        raise ArgumentTypeError(f"{path} nor {full_path} are not a valid path")
+    
+def file_arg_type(path):
+    full_path = get_full_path(path)
+    if not os.path.isfile(full_path):
+        raise ArgumentTypeError(f"{path} nor {full_path} are not a valid file")
+    return full_path
+
+def find_git_base_dir(starting_path=None):
+    if starting_path is None:
+        starting_path = os.getcwd()  
+    try:
+        base_dir = subprocess.check_output(
+            ['git', 'rev-parse', '--show-toplevel'],
+            cwd=starting_path,
+            text=True
+        ).strip()
+        return base_dir
+    except subprocess.CalledProcessError:
+        return None

services/tools/cli/base.py

@@ -1,21 +1,12 @@
 import sys
-import logging
-
-class BaseCLi:
-    def __init__(self):
-        self.logger = logging.getLogger(self.__class__.__name__)
-        handler = logging.StreamHandler()
-        formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
-        handler.setFormatter(formatter)
-        self.logger.addHandler(handler)
-        self.logger.setLevel(logging.ERROR)
 
+class BaseCli:
     def info(self, message):
-        self.logger.info(message)
+        print(f'[INFO] {message}')
 
     def warning(self, message):
-        self.logger.warning(message)
+        print(f'[WARNING] {message}')
 
     def error(self, message, exit_code=1):
-        self.logger.error(message)
+        print(f'[ERROR] {message}')
         sys.exit(exit_code)

services/tools/cli/constants.py

@@ -1,46 +1 @@
-PORTAINER_HOST_URLS = {
+SERVICES_DIR_RELATIVE_GIT_ROOT='services'
-    'vm-tools-100-55': 'https://portainer55.davydovcloud.com',
-    'vm-mixed-100-98': 'https://portainer98.davydovcloud.com'
-}
-
-PORTAINER_STACK_CLI_ACTIONS = {
-    'LIST': {
-        'value': 'list',
-        'arg_required': False,
-        'arg_name': '',
-    },
-    'CREATE': {
-        'value': 'create',
-        'arg_required': True,
-        'arg_name': 'stack-dir-path',
-    },
-    'REMOVE': {
-        'value': 'remove',
-        'arg_required': True,
-        'arg_name': 'stack-id',
-    },
-    'REDEPLOY': {
-        'value': 'redeploy',
-        'arg_required': True,
-        'arg_name': 'stack-id',
-    },
-    'START': {
-        'value': 'start',
-        'arg_required': True,
-        'arg_name': 'stack-id',
-    },
-    'STOP': {
-        'value': 'stop',
-        'arg_required': True,
-        'arg_name': 'stack-id',
-    },
-    'INFO': {
-        'value': 'info',
-        'arg_required': True,
-        'arg_name': 'stack-id',
-    },
-}
-
-DEFAULT_PORTAINER_JWT_CONFIG_FILE = '.portainer_cli_config'
-FUNCTIONAL_USER_USERNAME_ENV = 'PORTAINER_USERNAME'
-FUNCTIONAL_USER_PASSWORD_ENV = 'PORTAINER_PASSWORD'

services/tools/cli/dataclass/portainer/stack.py

@@ -1,87 +0,0 @@
-from dataclasses import dataclass, field, fields
-from typing import List, Optional
-
-@dataclass
-class Env:
-    name: Optional[str] = None
-    value: Optional[str] = None
-
-@dataclass
-class TeamAccess:
-    AccessLevel: Optional[int] = None
-    TeamId: Optional[int] = None
-
-@dataclass
-class UserAccess:
-    AccessLevel: Optional[int] = None
-    UserId: Optional[int] = None
-
-@dataclass
-class ResourceControl:
-    AccessLevel: Optional[int] = None
-    AdministratorsOnly: Optional[bool] = None
-    Id: Optional[int] = None
-    OwnerId: Optional[int] = None
-    Public: Optional[bool] = None
-    ResourceId: Optional[str] = None
-    SubResourceIds: List[str] = field(default_factory=list)
-    System: Optional[bool] = None
-    TeamAccesses: List[TeamAccess] = field(default_factory=list)
-    Type: Optional[int] = None
-    UserAccesses: List[UserAccess] = field(default_factory=list)
-
-@dataclass
-class AutoUpdate:
-    forcePullImage: Optional[bool] = None
-    forceUpdate: Optional[bool] = None
-    interval: Optional[str] = None
-    jobID: Optional[str] = None
-    webhook: Optional[str] = None
-
-@dataclass
-class GitAuthentication:
-    gitCredentialID: Optional[int] = None
-    password: Optional[str] = None
-    username: Optional[str] = None
-
-@dataclass
-class GitConfig:
-    authentication: GitAuthentication = GitAuthentication()
-    configFilePath: Optional[str] = None
-    configHash: Optional[str] = None
-    referenceName: Optional[str] = None
-    tlsskipVerify: Optional[bool] = None
-    url: Optional[str] = None
-
-@dataclass
-class Option:
-    prune: Optional[bool] = None
-
-@dataclass
-class PortainerStack:
-    AdditionalFiles: List[str] = field(default_factory=list)
-    AutoUpdate: AutoUpdate = AutoUpdate()
-    EndpointId: Optional[int] = None
-    EntryPoint: Optional[str] = None
-    Env: List['Env'] = field(default_factory=list)
-    Id: Optional[int] = None
-    Name: Optional[str] = None
-    Option: Option = Option()
-    ResourceControl: ResourceControl = ResourceControl()
-    Status: Optional[int] = None
-    SwarmId: Optional[str] = None
-    Type: Optional[int] = None
-    createdBy: Optional[str] = None
-    creationDate: Optional[int] = None
-    fromAppTemplate: Optional[bool] = None
-    gitConfig: GitConfig = GitConfig()
-    namespace: Optional[str] = None
-    projectPath: Optional[str] = None
-    updateDate: Optional[int] = None
-    updatedBy: Optional[str] = None
-    
-def classFromArgs(className, argDict):
-    fieldSet = {f.name for f in fields(className) if f.init}
-    filteredArgDict = {k : v for k, v in argDict.items() if k in fieldSet}
-    return className(**filteredArgDict)
-

services/tools/cli/dataclass/service.py

@@ -0,0 +1,14 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class Host:
+    ip: str
+    user: str
+
+@dataclass
+class Service:
+    name: str
+    host: Host
+    compose_file: str
+    env_file: str

services/tools/cli/docker_service.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+
+from argparse import ArgumentParser
+from args_utils import file_arg_type, find_git_base_dir
+from dataclass.service import Service, Host
+from base import BaseCli
+import yaml
+import subprocess
+from typing import List
+from constants import SERVICES_DIR_RELATIVE_GIT_ROOT
+
+CLI_DESCRIPTION = '''
+This CLI is meant to execute Docker commands for specified services
+defined in a services file. The services file can be supplied as an
+argument, with a default value of ../services.yaml.
+'''
+
+class DockerServiceCLI(BaseCli):
+
+    def configure_args(self, parser: ArgumentParser):
+        parser.add_argument(
+            '-f',
+            '--services-file',
+            help='Path to the services file',
+            default='services.yaml',
+            type=file_arg_type
+        )
+        parser.add_argument(
+            '-n',
+            '--service-name',
+            help='Name of the service to run (default: all)',
+            required=True,
+            type=str
+        )
+
+    def process_args(self, parser: ArgumentParser):
+        args = parser.parse_args()
+        return args
+
+    def __init__(self):
+        super().__init__()
+        parser = ArgumentParser(description=CLI_DESCRIPTION)
+        self.configure_args(parser)
+        self.args = self.process_args(parser)
+        self.services_basedir = find_git_base_dir() + f'/{SERVICES_DIR_RELATIVE_GIT_ROOT}'
+
+    def load_services(self) -> List[Service]:
+        with open(self.args.services_file, 'r') as file:
+            services_data = yaml.safe_load(file)
+            services = []
+            for service_data in services_data['services']:
+                host_data = service_data['host']
+                host = Host(ip=host_data['ip'], user=host_data['user'])
+                project_path = f"{self.services_basedir}/{service_data['name']}"
+                service = Service(
+                    name=service_data['name'],
+                    host=host,
+                    compose_file=f"{project_path}/{service_data['composeFile']}",
+                    env_file=f"{project_path}/{service_data['envFile']}"
+                )
+                services.append(service)
+            return services
+
+    def run_docker_command(self, service: Service):
+        command = [
+            'docker', '-H', f"{service.host.user}@{service.host.ip}",
+            'compose', '-f', f"{service.compose_file}",
+            '--env-file', f"{service.env_file}",
+            'up', '-d'
+        ]
+        
+        try:
+            subprocess.run(command, check=True)
+            self.info(f"Service: {service.name} running on host: {service.host.ip}")
+        except subprocess.CalledProcessError as e:
+            self.error(f"Error executing command: {' '.join(command)}, return code: {e.returncode}")
+
+
+    def start(self):
+        services = self.load_services()
+        if self.args.service_name == 'all':
+            for service in services:
+                self.run_docker_command(service)
+        else:
+            service = next((s for s in services if s.name == self.args.service_name), None)
+            if service:
+                self.run_docker_command(service)
+            else:
+                self.error(f'Error: Service "{self.args.service_name}" not found in the services file.')
+
+def main():
+    cli = DockerServiceCLI()
+    cli.start()
+
+if __name__ == "__main__":
+    main()

services/tools/cli/portainer_host.py

@@ -1,87 +0,0 @@
-import json
-from api import ApiClient
-from base import BaseCLi
-from constants import (
-    DEFAULT_PORTAINER_JWT_CONFIG_FILE,
-    FUNCTIONAL_USER_USERNAME_ENV,
-    FUNCTIONAL_USER_PASSWORD_ENV,
-)
-from dataclass.portainer.stack import PortainerStack, classFromArgs
-import os
-from typing import List
-import requests
-
-class MissingEnvironmentVariable(Exception):
-    pass
-
-class EmptyConfigException(Exception):
-    pass
-
-class PortainerHost(BaseCLi):
-
-    def __init__(self, host_url):
-        super().__init__()
-        self.host_url = host_url
-        api_url = f'{host_url}/api'
-        self.api = ApiClient(api_url=api_url)
-        self.jwt_config = {}
-        self.jwt_token = ''
-        home_directory = os.path.expanduser("~")
-        config_file_path = os.path.join(home_directory, DEFAULT_PORTAINER_JWT_CONFIG_FILE)
-        try:
-            self.__read_current_config(config_file_path)
-        except EmptyConfigException:
-            self.__create_config(config_file_path)
-            
-    def __read_current_config(self, file_path):
-        try:
-            with open(file_path, 'r') as file:
-                self.jwt_config = json.load(file)
-
-        except FileNotFoundError:
-            print(f"Configuration does not exist. Creating...")
-            raise EmptyConfigException()
-        try:
-            jwt_token = self.jwt_config[self.host_url]['jwt']
-            self.api.reassign_token(jwt_token)
-        except KeyError:
-            print(f"No token present in configuration file for host {self.host_url}. Creating...")
-            raise EmptyConfigException()
-            
-    def __create_config(self, file_path):
-        jwt_token = self.__get_jwt_token()
-        self.jwt_config[self.host_url] = {
-            "jwt": jwt_token,
-        }
-        self.api.reassign_token(jwt_token)
-        with open(file_path, 'w') as file:
-            json.dump(self.jwt_config, file, indent=4)
-
-    def __get_jwt_token(self):
-        def get_env(env_key):
-            try:
-                return os.environ[env_key]
-            except KeyError:
-                raise MissingEnvironmentVariable(f"{env_key} env does not exist")
-
-        payload = {
-            "Username": get_env(FUNCTIONAL_USER_USERNAME_ENV),
-            "Password": get_env(FUNCTIONAL_USER_PASSWORD_ENV)
-        }
-        
-        response = self.api.post('/auth', payload=payload)
-        try:
-            token = response.json()['jwt']
-        except KeyError:
-            self.error('No "jwt" key in /auth response found.')
-        return token
-
-    def list_stacks(self) -> List[PortainerStack]:
-        response = self.api.get('/stacks')
-        raw_stacks = response.json()
-        return [classFromArgs(PortainerStack, stack) for stack in raw_stacks]
-
-    
-        
-
-

services/tools/cli/portainer_stack.py

@@ -1,80 +0,0 @@
-#!/usr/bin/env python3
-
-from argparse import ArgumentParser
-from args_utils import dir_arg_type
-from base import BaseCLi
-from constants import PORTAINER_HOST_URLS, PORTAINER_STACK_CLI_ACTIONS
-from portainer_host import PortainerHost
-
-CLI_DESCRIPTION='''
-This cli is meant to be used to execute whoel LCM flow of
-Portainer host stacks. Portainer host can be supplied separately,
-or by the tag.
-'''
-
-class PortainerStackCLi(BaseCLi):
-
-    def configure_args(self, parser: ArgumentParser):
-        parser.add_argument(
-            '-a',
-            '--action',
-            help='Action to be performed on Portainer host',
-            required=True,
-            choices=[action['value'] for action in PORTAINER_STACK_CLI_ACTIONS.values()]
-        )
-        parser.add_argument(
-            '-s',
-            '--stack-dir-path',
-            help='Directory with the stack related files',
-            type=dir_arg_type
-        )
-        parser.add_argument(
-            '-i',
-            '--stack-id',
-            help='Id of the stack in Portainer (see list)',
-            type=int
-        )
-        parser.add_argument(
-            '-n',
-            '--hostname',
-            help='Portainer hostname to be used',
-            required=True,
-            type=str,
-            choices=PORTAINER_HOST_URLS.keys()
-        )
-
-
-    def process_args(self, parser: ArgumentParser):
-        args = parser.parse_args()
-        required_args = {
-            'stack-dir-path': args.stack_dir_path,
-            'stack-id': args.stack_id,
-        }
-
-        for action in PORTAINER_STACK_CLI_ACTIONS.values():
-            if not (args.action == action['value'] and action['arg_required']):
-                continue
-
-            for arg_name, arg_value in required_args.items():
-                if action['arg_name'] == arg_name and not arg_value:
-                    parser.print_help()
-                    self.error(f'Argument --{arg_name} is required.')
-        return args
-
-    def __init__(self):
-        super().__init__()
-        parser = ArgumentParser(description=CLI_DESCRIPTION)
-        self.configure_args(parser)
-        self.args = self.process_args(parser)
-
-    def start(self):
-        host = PortainerHost(PORTAINER_HOST_URLS[self.args.hostname])
-        print(host.list_stacks()[0])
-        
-
-def main():
-    cli = PortainerStackCLi()
-    cli.start()
-
-if __name__ == "__main__":
-    main()