Commit a477aef4 authored by Jerico Moeyersons's avatar Jerico Moeyersons 🏘
Browse files

Finished stack - needs to be tested

parent baf09a0a
# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.155.1/containers/python-3/.devcontainer/base.Dockerfile # See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.155.1/containers/python-3/.devcontainer/base.Dockerfile
# [Choice] Python version: 3, 3.9, 3.8, 3.7, 3.6 # [Choice] Python version: 3, 3.9, 3.8, 3.7, 3.6
ARG VARIANT="3" ARG VARIANT="3.9"
FROM mcr.microsoft.com/vscode/devcontainers/python:0-${VARIANT} FROM mcr.microsoft.com/vscode/devcontainers/python:0-${VARIANT}
# [Option] Install Node.js # [Option] Install Node.js
......
FROM python:3.9
#######################################
# Install ping and curl for debugging #
#######################################
RUN apt-get update
# install ping
RUN apt-get -y install iputils-ping curl
###############################
# Install Python dependencies #
###############################
# Add requirements file
ADD ./requirements.txt /tmp
WORKDIR /tmp
# Upgrade pip
RUN pip install --upgrade pip
# install requirements
RUN pip install -r requirements.txt
######################
# Start microservice #
######################
WORKDIR /usr/api
# Expose Port 80 for development
EXPOSE 80
CMD ["python", "./api.py"]
# Only for debugging
# CMD /bin/bash
...@@ -21,4 +21,51 @@ ch.setFormatter(formatter) ...@@ -21,4 +21,51 @@ ch.setFormatter(formatter)
api_log.addHandler(ch) api_log.addHandler(ch)
logging.info('API Service started on host ' + socket.gethostname()) logging.info('API Service started on host ' + socket.gethostname())
kernel = Kernel() kernel = Kernel()
\ No newline at end of file
#######
# API #
#######
@app.route('/')
def info():
return 'This is the API service'
@app.route("/spawn-workers/<number>", methods=["GET"])
def spawn_workers(number):
return kernel.spawn_workers(number)
@app.route('/start-worker/<number>', methods=["GET"])
def start_workers(number):
return kernel.multiple_start(number)
######################
# TEARDOWN FUNCTIONS #
######################
# These functions are called when the application needs to shut down
def shut_down_server():
"""Shuts down the Werkzeug simple server
Raises:
RuntimeError -- raised if the werkzeug server is not used.
"""
func = request.environ.get('werkzeug.server.shutdown')
if func is None:
raise RuntimeError('Not running with the Werkzeug Server')
func()
@app.route('/shutdown', methods=['POST'])
def shutdown():
"""Shuts down the server
Returns:
string -- notify that server is shutting down.
"""
shut_down_server()
return 'Server shutting down...'
def main():
app.run(host='0.0.0.0', port=80)
if __name__ == "__main__":
main()
__author__ = 'Jerico Moeyersons'
import logging
import http
class BrokenWorkerError(Exception):
"""BrokenPluginError: indicates that something is wrong with a plugin
Inherits from:
Exception
"""
status_code = http.HTTPStatus.NOT_IMPLEMENTED.value
def __init__(self, message=None):
""""Constructor
Keyword Arguments:
message {string} -- errormessage (default: {None})
"""
# Log that error happened.
logging.warn('BrokerWorkerError: ' + message)
self.message = message
self.status_code = BrokenWorkerError.status_code
class DockerError(Exception):
def __init__(self, message=None):
"""Constructor
Keyword Arguments:
message {string} -- errormessage (default: {None})
"""
# Log that error happened.
logging.warn('DockerError: ' + message)
self.message = message
...@@ -4,27 +4,74 @@ import http ...@@ -4,27 +4,74 @@ import http
import logging import logging
import sys import sys
import os import os
import docker
import requests import requests
import concurrent.futures import concurrent.futures
import exceptions
WORKER_URL = 'http://worker-' WORKER_URL = 'http://worker-'
DEFAULT_NETWORK = 'mandelbrot'
WORKER_IMAGE_NAME = 'worker'
class Kernel(object): class Kernel(object):
def __init__(self): def __init__(self):
logging.info('Initializing API Kernel...') logging.info('Initializing API Kernel...')
self.workers = os.getenv('WORKERS', 0) self.workers = []
self.next_worker = 0 self.next_worker = 0
if self.workers == 0:
logging.error('No workers defined')
sys.exit(1)
else:
self.next_worker = 1
self.active = False self.active = False
logging.info('Initializing docker client...')
try:
self.docker_client = docker.from_env()
except Exception:
logging.critical('An error with the docker clien occurred, exitting...')
sys.exit(1)
def spawn_workers(self, number=1):
# TODO: delete previous workers
for i in range(number):
# Try to start a container with the process
try:
# Start a new container with the plugin process.
container_name = WORKER_IMAGE_NAME + "-" + str(i)
container = self.docker_client.containers.run(
image=WORKER_IMAGE_NAME,
detach=True,
name=container_name,
network=DEFAULT_NETWORK
)
logging.debug('Container with name ' + container_name + ' - id - ' + container.id + ' started')
self.workers.append(container)
# If the container exits with a non-zero exit code
# something is wrong with the code of the plugin.
except docker.errors.ContainerError:
message = 'Container ' + container_name + ' behaving correctly, marking as broken'
raise exceptions.BrokenWorkerError(message)
# If no image is found, something is wrong with the installation of the plugin.
except docker.errors.ImageNotFound:
raise exceptions.BrokenWorkerError('Installation of worker ' + container_name+ ' is broken.')
# Something is wrong with Docker, exit system
except docker.errors.APIError:
logging.critical('Critical docker error, exiting...')
sys.exit(1)
message = "All workers spawned..."
code = http.HTTPStatus.OK.value
return (message, code)
def start_worker(self, worker=None): def start_worker(self, worker=None):
if worker is None: if worker is None:
url = WORKER_URL + self.next_worker + "/start" url = "http://" + str(self.workers[self.next_worker].name)
self.__set_next_worker() self.__set_next_worker()
else: else:
# TODO: check if worker exists # TODO: check if worker exists
...@@ -64,12 +111,16 @@ class Kernel(object): ...@@ -64,12 +111,16 @@ class Kernel(object):
else: else:
with concurrent.futures.ThreadPoolExecutor(max_workers=self.workers) as executor: with concurrent.futures.ThreadPoolExecutor(max_workers=self.workers) as executor:
executor.submit(self.start_worker, self) executor.submit(self.start_worker, self)
return "All workers started and finished..." message = "All workers started and finished..."
code = http.HTTPStatus.OK.value
return (message, code)
else: else:
return "Another job is running, please try again later..." code = http.HTTPStatus.IM_A_TEAPOT.value
message = "Another job is running, please try again later..."
return (message, code)
def __set_next_worker(self): def __set_next_worker(self):
if self.next_worker == self.workers: if self.next_worker == length(self.workers)-1:
self.next_worker = 1 self.next_worker = 1
else: else:
self.next_worker += 1 self.next_worker += 1
flask flask
\ No newline at end of file docker
\ No newline at end of file
__author__ = 'Jerico Moeyersons'
import sys
import os
from subprocess import call
import logging
import http
import click
import requests
import docker
API_URL = 'http://localhost:8000'
# Logging for debug information
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
logging.captureWarnings(True)
# Use click.echo instead of print
@click.group()
@click.option('--verbose', is_flag=True,
help='Verbose prints more output which is used for debugging')
def cli(verbose):
if verbose:
click.echo('verbose mode active')
##################
# Start and stop #
##################
@cli.command()
def on():
"""This script starts the application
This command has to be called first, otherwise other commands won't work
"""
call(["docker-compose", "up", "-d"])
logging.info('Mandelbrot-creator started')
click.echo('Mandelbrot-creator started!')
@cli.command()
def off():
"""This script stops the application
Call this command to stop the application gracefully
"""
# Also call the stop command to the API which shuts down all workers
url = API_URL + '/shutdown'
click.echo('Shutting down everything...')
# TODO Error handling if application is not started.
r = requests.post(url)
call(["docker-compose", "down"])
logging.info('Mandelbrot-creator shut down')
click.echo('Mandelbrot-creator shut down')
###################
# Plugins command #
###################
@cli.command()
@click.argument("number")
def start_workers(number):
"""Start the workers
"""
url = API_URL + '/start-worker/' + str(number)
try:
r = requests.get(url)
click.echo(r.text)
except requests.ConnectionError:
logging.error('Connection error in function start_workers')
click.echo('ERROR: Could not connect to the mandelbrot-creator application.\n \
Are you sure the application is started?')
except requests.Timeout:
logging.error('Timeout error in function start_workers')
click.echo('Worker took to long to respond, please try the command again.')
@cli.command()
@click.argument("number")
def spawn_workers(number):
url = API_URL + "/spawn-workers/" + str(number)
try:
r = requests.get(url)
click.echo(r.text)
except requests.ConnectionError:
logging.error('Connection error in function spawn_workers')
click.echo('ERROR: Could not connect to the mandelbrot-creator application.\n \
Are you sure the application is started?')
except requests.Timeout:
logging.error('Timeout error in function spawn_workers')
click.echo('Worker took to long to respond, please try the command again.')
click
requests
docker
docker-compose
from setuptools import setup
setup(
name='mandelbrot-creator',
version='1.0',
py_modules=['mandelbrot-creator'],
install_requires=[
'Click', 'requests', 'docker', 'docker-compose'
],
entry_points='''
[console_scripts]
mandelbrot-creator=mandelbrot_creator:cli
'''
)
version: "3.5"
services:
api:
container_name: api
build:
context: ./API
dockerfile: Dockerfile
ports:
- "8080:80"
volumes:
- "./API/:/usr/api/"
- "/var/run/docker.sock:/var/run/docker.sock"
networks:
- mandelbrot
networks:
mandelbrot:
FROM python:3.9
#######################################
# Install ping and curl for debugging #
#######################################
RUN apt-get update
# install ping
RUN apt-get -y install iputils-ping curl
###############################
# Install Python dependencies #
###############################
# Add requirements file
ADD ./requirements.txt /tmp
WORKDIR /tmp
# Upgrade pip
RUN pip install --upgrade pip
# install requirements
RUN pip install -r requirements.txt
######################
# Start microservice #
######################
WORKDIR /usr/worker
COPY . .
# Expose Port 80 for development
EXPOSE 80
CMD ["python", "./api.py"]
# Only for debugging
# CMD /bin/bash
__author__ = 'Jerico Moeyersons'
import logging
import os
import sys
import socket
from kernel import Kernel
from flask import Flask, request, jsonify
import validation
import exceptions
app = Flask(__name__)
logging.basicConfig(filename='worker.log', level=logging.WARNING)
logging.captureWarnings(True)
worker_log = logging.getLogger()
worker_log.setLevel(logging.DEBUG)
ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
worker_log.addHandler(ch)
logging.info('Worker Service started on host ' + socket.gethostname())
# Init Kernel
kernel = Kernel()
#######
# API #
#######
@app.route('/')
def info():
return 'This is the worker service'
@app.route('/start', methods=['POST'])
def start_worker():
# POST handler
if request.method == "POST":
# Parse payload
payload = request.get_json()
validation.val_start_worker(payload)
return kernel.start()
######################
# TEARDOWN FUNCTIONS #
######################
# These functions are called when the application needs to shut down
def shut_down_server():
"""Shuts down the Werkzeug simple server
Raises:
RuntimeError -- raised if the werkzeug server is not used.
"""
func = request.environ.get('werkzeug.server.shutdown')
if func is None:
raise RuntimeError('Not running with the Werkzeug Server')
func()
@app.route('/shutdown', methods=['POST'])
def shutdown():
"""Shuts down the server
Returns:
string -- notify that server is shutting down.
"""
shut_down_server()
return 'Server shutting down...'
def main():
app.run(host='0.0.0.0', port=80)
if __name__ == "__main__":
main()
__author__ = 'Jerico Moeyersons'
import logging
import http
class InvalidPayloadError(Exception):
"""Invalid payload exception
Indicates that the payload received by the microservice was invalid.
See message for more details
Inherits from exception:
Exception
"""
# HTTP status code
status_code = http.HTTPStatus.BAD_REQUEST.value
def __init__(self, message=None, status_code=None, payload=None, validation_message=None):
"""Constructor
Arguments:
message {string} -- Message for the exception
Keyword Arguments:
status_code {integer} -- HTTP status code (default: {None})
payload {object} -- Response payload (default: {None})
"""
Exception.__init__(self)
logging.warn('InvalidPayloadError: ' + message + '\n' + str(payload))
self.message = message
self.payload = payload
if status_code:
self.status_code = status_code
else:
self.status_code = InvalidPayloadError.status_code
self.validation_message = validation_message
def to_dict(self):
rv = {}
rv['received-payload'] = self.payload or ()
rv['message'] = self.message
rv['vaidation_message'] = self.validation_message
return rv
...@@ -3,6 +3,7 @@ __author__ = 'Jerico Moeyersons' ...@@ -3,6 +3,7 @@ __author__ = 'Jerico Moeyersons'
import sys import sys
import math import math
import numpy as np import numpy as np
import http
from PIL import Image from PIL import Image
from itertools import repeat from itertools import repeat
from multiprocessing import Pool from multiprocessing import Pool
...@@ -30,6 +31,11 @@ class Kernel(object): ...@@ -30,6 +31,11 @@ class Kernel(object):
mandelbrot.save(f'{name}.png') mandelbrot.save(f'{name}.png')
print(f'saved {name}.png') print(f'saved {name}.png')
message = 'Mandelbrot calculated and saved succesfully!'
code = http.HTTPStatus.OK.value
return (message, code)
def get_col(self, args): def get_col(self, args):
iy, width, height, max_iterations = args iy, width, height, max_iterations = args
......
numpy numpy
pillows pillow
\ No newline at end of file flask
jsonschema
\ No newline at end of file
__author__ = 'Jerico Moeyersons'
from jsonschema import validate
from jsonschema import ValidationError
################
# JSON schemas #
################