diff --git a/services/audiobookshelf/init.sh b/services/audiobookshelf/init.sh deleted file mode 100755 index 1520a56..0000000 --- a/services/audiobookshelf/init.sh +++ /dev/null @@ -1,10 +0,0 @@ -#! /usr/bin/bash - -source ./.env -set -xe - -mkdir -p "${DOCKER_PARENT_PATH}" -mkdir -p "${SVC_PATH}" -mkdir -p "${SVC_PATH}/config" -mkdir -p "${SVC_PATH}/metadata" - diff --git a/services/ippoolmanager-sql-server /.env b/services/ippoolmanager-sql-server /.env deleted file mode 100644 index 5d6ace1..0000000 --- a/services/ippoolmanager-sql-server /.env +++ /dev/null @@ -1,2 +0,0 @@ -MYSQL_ROOT_PASSWORD="2F&B7yUoaE9F" -MYSQL_USER_PASSWORD="ippoolmanager" \ No newline at end of file diff --git a/services/ippoolmanager-sql-server/.env b/services/ippoolmanager-sql-server/.env new file mode 100644 index 0000000..b8e0efd --- /dev/null +++ b/services/ippoolmanager-sql-server/.env @@ -0,0 +1,4 @@ +MYSQL_ROOT_PASSWORD="2F&B7yUoaE9F" +MYSQL_USER_PASSWORD="ippoolmanager" +USER_PASSWORD="wxCD2JKXWbjCyBd504Q43Uv2W6gg80Y671" +ROOT_PASSWORD="FMB2u7Lhgt282rYPW914J1ivg1bn5YPt0X" \ No newline at end of file diff --git a/services/ippoolmanager-sql-server /docker-compose.yml b/services/ippoolmanager-sql-server/docker-compose.yml similarity index 86% rename from services/ippoolmanager-sql-server /docker-compose.yml rename to services/ippoolmanager-sql-server/docker-compose.yml index 850f64b..a231372 100644 --- a/services/ippoolmanager-sql-server /docker-compose.yml +++ b/services/ippoolmanager-sql-server/docker-compose.yml @@ -1,4 +1,3 @@ -version: '3.3' services: db: image: mysql:latest @@ -9,7 +8,7 @@ services: MYSQL_PASSWORD: '${USER_PASSWORD}' MYSQL_ROOT_PASSWORD: '${ROOT_PASSWORD}' ports: - - '192.168.100.57:3306:3306' + - '3306:3306' expose: - '3306' volumes: diff --git a/services/jenkins/ci/jobs/modules/Utils.groovy b/services/jenkins/ci/jobs/modules/Utils.groovy new file mode 100644 index 0000000..af5963c --- /dev/null +++ b/services/jenkins/ci/jobs/modules/Utils.groovy @@ -0,0 +1,43 @@ +#!/usr/bin/env groovy + +package modules + +import javaposse.jobdsl.dsl.Job + +public class Utils { + static Job createJob( + String jobName, + String jobDescription, + jobDslCtx + ) { + Job job = jobDslCtx.pipelineJob(jobName) + job.with { + description(jobDescription) + } + return job + } + + static void addGiteaRepository( + Job job, + String giteaRepositoryURL, + String jenkinsFilePath + ) { + job.with { + definition { + cpsScm { + lightweight(false) + scm { + git { + branch('main') + remote { + url(giteaRepositoryURL) + credentials('77c771b9-20f4-426d-8f1e-ed901afe9eb9') + } + } + } + scriptPath(jenkinsFilePath) + } + } + } + } +} \ No newline at end of file diff --git a/services/jenkins/ci/jobs/seed.groovy b/services/jenkins/ci/jobs/seed.groovy new file mode 100644 index 0000000..de17c5d --- /dev/null +++ b/services/jenkins/ci/jobs/seed.groovy @@ -0,0 +1,18 @@ +#!/usr/bin/env groovy +import javaposse.jobdsl.dsl.Job +import modules.Utils + +void createSeed() { + String jobName = 'seed' + String jobDescription = 'This job creates all other jobs' + String jenkinsFile = 'services/jenkins/ci/pipelines/seed.groovy' + String repoUrl = 'https://git.tylencloud.com/tylen/andromeda-setup' + + Job job = Utils.createJob(jobName, jobDescription, this) + Utils.addGiteaRepository( + job, + repoUrl, + jenkinsFile + ) + +} \ No newline at end of file diff --git a/services/jenkins/ci/pipelines/seed.groovy b/services/jenkins/ci/pipelines/seed.groovy new file mode 100644 index 0000000..e69de29 diff --git a/services/services.env b/services/services.env deleted file mode 100755 index 30219e4..0000000 --- a/services/services.env +++ /dev/null @@ -1 +0,0 @@ -# Environment variables generated from JSON data diff --git a/services/services.yaml b/services/services.yaml new file mode 100644 index 0000000..5e8e1ed --- /dev/null +++ b/services/services.yaml @@ -0,0 +1,22 @@ +defaultServiceValues: &defaultServiceValues + composeFile: "docker-compose.yml" + envFile: ".env" + +vm-tools-100-65: &vm-tools-100-65 + ip: "192.168.100.65" + user: vm-user + +vm-media-100-55: &vm-media-100-55 + ip: "192.168.100.55" + user: vm-user + +services: + - name: "audiobookshelf" + host: + <<: *vm-media-100-55 + <<: *defaultServiceValues + + - name: "ippoolmanager-sql-server" + host: + <<: *vm-tools-100-65 + <<: *defaultServiceValues diff --git a/services/tools/cli/args_utils.py b/services/tools/cli/args_utils.py index 535f0b7..89cd6ab 100644 --- a/services/tools/cli/args_utils.py +++ b/services/tools/cli/args_utils.py @@ -1,5 +1,6 @@ from argparse import ArgumentTypeError import os +import subprocess def get_full_path(relative_path): if not os.path.isabs(relative_path): @@ -15,4 +16,23 @@ def dir_arg_type(path): if os.path.isdir(full_path): return full_path else: - raise ArgumentTypeError(f"{path} nor {full_path} are not a valid path") \ No newline at end of file + raise ArgumentTypeError(f"{path} nor {full_path} are not a valid path") + +def file_arg_type(path): + full_path = get_full_path(path) + if not os.path.isfile(full_path): + raise ArgumentTypeError(f"{path} nor {full_path} are not a valid file") + return full_path + +def find_git_base_dir(starting_path=None): + if starting_path is None: + starting_path = os.getcwd() + try: + base_dir = subprocess.check_output( + ['git', 'rev-parse', '--show-toplevel'], + cwd=starting_path, + text=True + ).strip() + return base_dir + except subprocess.CalledProcessError: + return None \ No newline at end of file diff --git a/services/tools/cli/base.py b/services/tools/cli/base.py index f5175ad..15ee39e 100644 --- a/services/tools/cli/base.py +++ b/services/tools/cli/base.py @@ -1,21 +1,12 @@ import sys -import logging - -class BaseCLi: - def __init__(self): - self.logger = logging.getLogger(self.__class__.__name__) - handler = logging.StreamHandler() - formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') - handler.setFormatter(formatter) - self.logger.addHandler(handler) - self.logger.setLevel(logging.ERROR) +class BaseCli: def info(self, message): - self.logger.info(message) + print(f'[INFO] {message}') def warning(self, message): - self.logger.warning(message) + print(f'[WARNING] {message}') def error(self, message, exit_code=1): - self.logger.error(message) + print(f'[ERROR] {message}') sys.exit(exit_code) \ No newline at end of file diff --git a/services/tools/cli/constants.py b/services/tools/cli/constants.py index c30d351..61bb83b 100644 --- a/services/tools/cli/constants.py +++ b/services/tools/cli/constants.py @@ -1,46 +1 @@ -PORTAINER_HOST_URLS = { - 'vm-tools-100-55': 'https://portainer55.davydovcloud.com', - 'vm-mixed-100-98': 'https://portainer98.davydovcloud.com' -} - -PORTAINER_STACK_CLI_ACTIONS = { - 'LIST': { - 'value': 'list', - 'arg_required': False, - 'arg_name': '', - }, - 'CREATE': { - 'value': 'create', - 'arg_required': True, - 'arg_name': 'stack-dir-path', - }, - 'REMOVE': { - 'value': 'remove', - 'arg_required': True, - 'arg_name': 'stack-id', - }, - 'REDEPLOY': { - 'value': 'redeploy', - 'arg_required': True, - 'arg_name': 'stack-id', - }, - 'START': { - 'value': 'start', - 'arg_required': True, - 'arg_name': 'stack-id', - }, - 'STOP': { - 'value': 'stop', - 'arg_required': True, - 'arg_name': 'stack-id', - }, - 'INFO': { - 'value': 'info', - 'arg_required': True, - 'arg_name': 'stack-id', - }, -} - -DEFAULT_PORTAINER_JWT_CONFIG_FILE = '.portainer_cli_config' -FUNCTIONAL_USER_USERNAME_ENV = 'PORTAINER_USERNAME' -FUNCTIONAL_USER_PASSWORD_ENV = 'PORTAINER_PASSWORD' \ No newline at end of file +SERVICES_DIR_RELATIVE_GIT_ROOT='services' \ No newline at end of file diff --git a/services/tools/cli/dataclass/portainer/stack.py b/services/tools/cli/dataclass/portainer/stack.py deleted file mode 100644 index 550d653..0000000 --- a/services/tools/cli/dataclass/portainer/stack.py +++ /dev/null @@ -1,87 +0,0 @@ -from dataclasses import dataclass, field, fields -from typing import List, Optional - -@dataclass -class Env: - name: Optional[str] = None - value: Optional[str] = None - -@dataclass -class TeamAccess: - AccessLevel: Optional[int] = None - TeamId: Optional[int] = None - -@dataclass -class UserAccess: - AccessLevel: Optional[int] = None - UserId: Optional[int] = None - -@dataclass -class ResourceControl: - AccessLevel: Optional[int] = None - AdministratorsOnly: Optional[bool] = None - Id: Optional[int] = None - OwnerId: Optional[int] = None - Public: Optional[bool] = None - ResourceId: Optional[str] = None - SubResourceIds: List[str] = field(default_factory=list) - System: Optional[bool] = None - TeamAccesses: List[TeamAccess] = field(default_factory=list) - Type: Optional[int] = None - UserAccesses: List[UserAccess] = field(default_factory=list) - -@dataclass -class AutoUpdate: - forcePullImage: Optional[bool] = None - forceUpdate: Optional[bool] = None - interval: Optional[str] = None - jobID: Optional[str] = None - webhook: Optional[str] = None - -@dataclass -class GitAuthentication: - gitCredentialID: Optional[int] = None - password: Optional[str] = None - username: Optional[str] = None - -@dataclass -class GitConfig: - authentication: GitAuthentication = GitAuthentication() - configFilePath: Optional[str] = None - configHash: Optional[str] = None - referenceName: Optional[str] = None - tlsskipVerify: Optional[bool] = None - url: Optional[str] = None - -@dataclass -class Option: - prune: Optional[bool] = None - -@dataclass -class PortainerStack: - AdditionalFiles: List[str] = field(default_factory=list) - AutoUpdate: AutoUpdate = AutoUpdate() - EndpointId: Optional[int] = None - EntryPoint: Optional[str] = None - Env: List['Env'] = field(default_factory=list) - Id: Optional[int] = None - Name: Optional[str] = None - Option: Option = Option() - ResourceControl: ResourceControl = ResourceControl() - Status: Optional[int] = None - SwarmId: Optional[str] = None - Type: Optional[int] = None - createdBy: Optional[str] = None - creationDate: Optional[int] = None - fromAppTemplate: Optional[bool] = None - gitConfig: GitConfig = GitConfig() - namespace: Optional[str] = None - projectPath: Optional[str] = None - updateDate: Optional[int] = None - updatedBy: Optional[str] = None - -def classFromArgs(className, argDict): - fieldSet = {f.name for f in fields(className) if f.init} - filteredArgDict = {k : v for k, v in argDict.items() if k in fieldSet} - return className(**filteredArgDict) - diff --git a/services/tools/cli/dataclass/service.py b/services/tools/cli/dataclass/service.py new file mode 100644 index 0000000..6cedb0a --- /dev/null +++ b/services/tools/cli/dataclass/service.py @@ -0,0 +1,14 @@ +from dataclasses import dataclass + + +@dataclass +class Host: + ip: str + user: str + +@dataclass +class Service: + name: str + host: Host + compose_file: str + env_file: str \ No newline at end of file diff --git a/services/tools/cli/docker_service.py b/services/tools/cli/docker_service.py new file mode 100755 index 0000000..d6624bc --- /dev/null +++ b/services/tools/cli/docker_service.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 + +from argparse import ArgumentParser +from args_utils import file_arg_type, find_git_base_dir +from dataclass.service import Service, Host +from base import BaseCli +import yaml +import subprocess +from typing import List +from constants import SERVICES_DIR_RELATIVE_GIT_ROOT + +CLI_DESCRIPTION = ''' +This CLI is meant to execute Docker commands for specified services +defined in a services file. The services file can be supplied as an +argument, with a default value of ../services.yaml. +''' + +class DockerServiceCLI(BaseCli): + + def configure_args(self, parser: ArgumentParser): + parser.add_argument( + '-f', + '--services-file', + help='Path to the services file', + default='services.yaml', + type=file_arg_type + ) + parser.add_argument( + '-n', + '--service-name', + help='Name of the service to run (default: all)', + required=True, + type=str + ) + + def process_args(self, parser: ArgumentParser): + args = parser.parse_args() + return args + + def __init__(self): + super().__init__() + parser = ArgumentParser(description=CLI_DESCRIPTION) + self.configure_args(parser) + self.args = self.process_args(parser) + self.services_basedir = find_git_base_dir() + f'/{SERVICES_DIR_RELATIVE_GIT_ROOT}' + + def load_services(self) -> List[Service]: + with open(self.args.services_file, 'r') as file: + services_data = yaml.safe_load(file) + services = [] + for service_data in services_data['services']: + host_data = service_data['host'] + host = Host(ip=host_data['ip'], user=host_data['user']) + project_path = f"{self.services_basedir}/{service_data['name']}" + service = Service( + name=service_data['name'], + host=host, + compose_file=f"{project_path}/{service_data['composeFile']}", + env_file=f"{project_path}/{service_data['envFile']}" + ) + services.append(service) + return services + + def run_docker_command(self, service: Service): + command = [ + 'docker', '-H', f"{service.host.user}@{service.host.ip}", + 'compose', '-f', f"{service.compose_file}", + '--env-file', f"{service.env_file}", + 'up', '-d' + ] + + try: + subprocess.run(command, check=True) + self.info(f"Service: {service.name} running on host: {service.host.ip}") + except subprocess.CalledProcessError as e: + self.error(f"Error executing command: {' '.join(command)}, return code: {e.returncode}") + + + def start(self): + services = self.load_services() + if self.args.service_name == 'all': + for service in services: + self.run_docker_command(service) + else: + service = next((s for s in services if s.name == self.args.service_name), None) + if service: + self.run_docker_command(service) + else: + self.error(f'Error: Service "{self.args.service_name}" not found in the services file.') + +def main(): + cli = DockerServiceCLI() + cli.start() + +if __name__ == "__main__": + main() diff --git a/services/tools/cli/portainer_host.py b/services/tools/cli/portainer_host.py deleted file mode 100644 index a370d60..0000000 --- a/services/tools/cli/portainer_host.py +++ /dev/null @@ -1,87 +0,0 @@ -import json -from api import ApiClient -from base import BaseCLi -from constants import ( - DEFAULT_PORTAINER_JWT_CONFIG_FILE, - FUNCTIONAL_USER_USERNAME_ENV, - FUNCTIONAL_USER_PASSWORD_ENV, -) -from dataclass.portainer.stack import PortainerStack, classFromArgs -import os -from typing import List -import requests - -class MissingEnvironmentVariable(Exception): - pass - -class EmptyConfigException(Exception): - pass - -class PortainerHost(BaseCLi): - - def __init__(self, host_url): - super().__init__() - self.host_url = host_url - api_url = f'{host_url}/api' - self.api = ApiClient(api_url=api_url) - self.jwt_config = {} - self.jwt_token = '' - home_directory = os.path.expanduser("~") - config_file_path = os.path.join(home_directory, DEFAULT_PORTAINER_JWT_CONFIG_FILE) - try: - self.__read_current_config(config_file_path) - except EmptyConfigException: - self.__create_config(config_file_path) - - def __read_current_config(self, file_path): - try: - with open(file_path, 'r') as file: - self.jwt_config = json.load(file) - - except FileNotFoundError: - print(f"Configuration does not exist. Creating...") - raise EmptyConfigException() - try: - jwt_token = self.jwt_config[self.host_url]['jwt'] - self.api.reassign_token(jwt_token) - except KeyError: - print(f"No token present in configuration file for host {self.host_url}. Creating...") - raise EmptyConfigException() - - def __create_config(self, file_path): - jwt_token = self.__get_jwt_token() - self.jwt_config[self.host_url] = { - "jwt": jwt_token, - } - self.api.reassign_token(jwt_token) - with open(file_path, 'w') as file: - json.dump(self.jwt_config, file, indent=4) - - def __get_jwt_token(self): - def get_env(env_key): - try: - return os.environ[env_key] - except KeyError: - raise MissingEnvironmentVariable(f"{env_key} env does not exist") - - payload = { - "Username": get_env(FUNCTIONAL_USER_USERNAME_ENV), - "Password": get_env(FUNCTIONAL_USER_PASSWORD_ENV) - } - - response = self.api.post('/auth', payload=payload) - try: - token = response.json()['jwt'] - except KeyError: - self.error('No "jwt" key in /auth response found.') - return token - - def list_stacks(self) -> List[PortainerStack]: - response = self.api.get('/stacks') - raw_stacks = response.json() - return [classFromArgs(PortainerStack, stack) for stack in raw_stacks] - - - - - diff --git a/services/tools/cli/portainer_stack.py b/services/tools/cli/portainer_stack.py deleted file mode 100755 index c69ef81..0000000 --- a/services/tools/cli/portainer_stack.py +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env python3 - -from argparse import ArgumentParser -from args_utils import dir_arg_type -from base import BaseCLi -from constants import PORTAINER_HOST_URLS, PORTAINER_STACK_CLI_ACTIONS -from portainer_host import PortainerHost - -CLI_DESCRIPTION=''' -This cli is meant to be used to execute whoel LCM flow of -Portainer host stacks. Portainer host can be supplied separately, -or by the tag. -''' - -class PortainerStackCLi(BaseCLi): - - def configure_args(self, parser: ArgumentParser): - parser.add_argument( - '-a', - '--action', - help='Action to be performed on Portainer host', - required=True, - choices=[action['value'] for action in PORTAINER_STACK_CLI_ACTIONS.values()] - ) - parser.add_argument( - '-s', - '--stack-dir-path', - help='Directory with the stack related files', - type=dir_arg_type - ) - parser.add_argument( - '-i', - '--stack-id', - help='Id of the stack in Portainer (see list)', - type=int - ) - parser.add_argument( - '-n', - '--hostname', - help='Portainer hostname to be used', - required=True, - type=str, - choices=PORTAINER_HOST_URLS.keys() - ) - - - def process_args(self, parser: ArgumentParser): - args = parser.parse_args() - required_args = { - 'stack-dir-path': args.stack_dir_path, - 'stack-id': args.stack_id, - } - - for action in PORTAINER_STACK_CLI_ACTIONS.values(): - if not (args.action == action['value'] and action['arg_required']): - continue - - for arg_name, arg_value in required_args.items(): - if action['arg_name'] == arg_name and not arg_value: - parser.print_help() - self.error(f'Argument --{arg_name} is required.') - return args - - def __init__(self): - super().__init__() - parser = ArgumentParser(description=CLI_DESCRIPTION) - self.configure_args(parser) - self.args = self.process_args(parser) - - def start(self): - host = PortainerHost(PORTAINER_HOST_URLS[self.args.hostname]) - print(host.list_stacks()[0]) - - -def main(): - cli = PortainerStackCLi() - cli.start() - -if __name__ == "__main__": - main() \ No newline at end of file