Commit a477aef4 authored by Jerico Moeyersons 's avatar Jerico Moeyersons 🏘
Browse files

Finished stack - needs to be tested

parent baf09a0a
# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.155.1/containers/python-3/.devcontainer/base.Dockerfile
# [Choice] Python version: 3, 3.9, 3.8, 3.7, 3.6
ARG VARIANT="3"
ARG VARIANT="3.9"
FROM mcr.microsoft.com/vscode/devcontainers/python:0-${VARIANT}
# [Option] Install Node.js
......
FROM python:3.9
#######################################
# Install ping and curl for debugging #
#######################################
RUN apt-get update
# install ping
RUN apt-get -y install iputils-ping curl
###############################
# Install Python dependencies #
###############################
# Add requirements file
ADD ./requirements.txt /tmp
WORKDIR /tmp
# Upgrade pip
RUN pip install --upgrade pip
# install requirements
RUN pip install -r requirements.txt
######################
# Start microservice #
######################
WORKDIR /usr/api
# Expose Port 80 for development
EXPOSE 80
CMD ["python", "./api.py"]
# Only for debugging
# CMD /bin/bash
......@@ -21,4 +21,51 @@ ch.setFormatter(formatter)
api_log.addHandler(ch)
logging.info('API Service started on host ' + socket.gethostname())
kernel = Kernel()
\ No newline at end of file
kernel = Kernel()
#######
# API #
#######
@app.route('/')
def info():
return 'This is the API service'
@app.route("/spawn-workers/<number>", methods=["GET"])
def spawn_workers(number):
return kernel.spawn_workers(number)
@app.route('/start-worker/<number>', methods=["GET"])
def start_workers(number):
return kernel.multiple_start(number)
######################
# TEARDOWN FUNCTIONS #
######################
# These functions are called when the application needs to shut down
def shut_down_server():
"""Shuts down the Werkzeug simple server
Raises:
RuntimeError -- raised if the werkzeug server is not used.
"""
func = request.environ.get('werkzeug.server.shutdown')
if func is None:
raise RuntimeError('Not running with the Werkzeug Server')
func()
@app.route('/shutdown', methods=['POST'])
def shutdown():
"""Shuts down the server
Returns:
string -- notify that server is shutting down.
"""
shut_down_server()
return 'Server shutting down...'
def main():
app.run(host='0.0.0.0', port=80)
if __name__ == "__main__":
main()
__author__ = 'Jerico Moeyersons'
import logging
import http
class BrokenWorkerError(Exception):
"""BrokenPluginError: indicates that something is wrong with a plugin
Inherits from:
Exception
"""
status_code = http.HTTPStatus.NOT_IMPLEMENTED.value
def __init__(self, message=None):
""""Constructor
Keyword Arguments:
message {string} -- errormessage (default: {None})
"""
# Log that error happened.
logging.warn('BrokerWorkerError: ' + message)
self.message = message
self.status_code = BrokenWorkerError.status_code
class DockerError(Exception):
def __init__(self, message=None):
"""Constructor
Keyword Arguments:
message {string} -- errormessage (default: {None})
"""
# Log that error happened.
logging.warn('DockerError: ' + message)
self.message = message
......@@ -4,27 +4,74 @@ import http
import logging
import sys
import os
import docker
import requests
import concurrent.futures
import exceptions
WORKER_URL = 'http://worker-'
DEFAULT_NETWORK = 'mandelbrot'
WORKER_IMAGE_NAME = 'worker'
class Kernel(object):
def __init__(self):
logging.info('Initializing API Kernel...')
self.workers = os.getenv('WORKERS', 0)
self.workers = []
self.next_worker = 0
if self.workers == 0:
logging.error('No workers defined')
sys.exit(1)
else:
self.next_worker = 1
self.active = False
logging.info('Initializing docker client...')
try:
self.docker_client = docker.from_env()
except Exception:
logging.critical('An error with the docker clien occurred, exitting...')
sys.exit(1)
def spawn_workers(self, number=1):
# TODO: delete previous workers
for i in range(number):
# Try to start a container with the process
try:
# Start a new container with the plugin process.
container_name = WORKER_IMAGE_NAME + "-" + str(i)
container = self.docker_client.containers.run(
image=WORKER_IMAGE_NAME,
detach=True,
name=container_name,
network=DEFAULT_NETWORK
)
logging.debug('Container with name ' + container_name + ' - id - ' + container.id + ' started')
self.workers.append(container)
# If the container exits with a non-zero exit code
# something is wrong with the code of the plugin.
except docker.errors.ContainerError:
message = 'Container ' + container_name + ' behaving correctly, marking as broken'
raise exceptions.BrokenWorkerError(message)
# If no image is found, something is wrong with the installation of the plugin.
except docker.errors.ImageNotFound:
raise exceptions.BrokenWorkerError('Installation of worker ' + container_name+ ' is broken.')
# Something is wrong with Docker, exit system
except docker.errors.APIError:
logging.critical('Critical docker error, exiting...')
sys.exit(1)
message = "All workers spawned..."
code = http.HTTPStatus.OK.value
return (message, code)
def start_worker(self, worker=None):
if worker is None:
url = WORKER_URL + self.next_worker + "/start"
url = "http://" + str(self.workers[self.next_worker].name)
self.__set_next_worker()
else:
# TODO: check if worker exists
......@@ -64,12 +111,16 @@ class Kernel(object):
else:
with concurrent.futures.ThreadPoolExecutor(max_workers=self.workers) as executor:
executor.submit(self.start_worker, self)
return "All workers started and finished..."
message = "All workers started and finished..."
code = http.HTTPStatus.OK.value
return (message, code)
else:
return "Another job is running, please try again later..."
code = http.HTTPStatus.IM_A_TEAPOT.value
message = "Another job is running, please try again later..."
return (message, code)
def __set_next_worker(self):
if self.next_worker == self.workers:
if self.next_worker == length(self.workers)-1:
self.next_worker = 1
else:
self.next_worker += 1
flask
\ No newline at end of file
flask
docker
\ No newline at end of file
__author__ = 'Jerico Moeyersons'
import sys
import os
from subprocess import call
import logging
import http
import click
import requests
import docker
API_URL = 'http://localhost:8000'
# Logging for debug information
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
logging.captureWarnings(True)
# Use click.echo instead of print
@click.group()
@click.option('--verbose', is_flag=True,
help='Verbose prints more output which is used for debugging')
def cli(verbose):
if verbose:
click.echo('verbose mode active')
##################
# Start and stop #
##################
@cli.command()
def on():
"""This script starts the application
This command has to be called first, otherwise other commands won't work
"""
call(["docker-compose", "up", "-d"])
logging.info('Mandelbrot-creator started')
click.echo('Mandelbrot-creator started!')
@cli.command()
def off():
"""This script stops the application
Call this command to stop the application gracefully
"""
# Also call the stop command to the API which shuts down all workers
url = API_URL + '/shutdown'
click.echo('Shutting down everything...')
# TODO Error handling if application is not started.
r = requests.post(url)
call(["docker-compose", "down"])
logging.info('Mandelbrot-creator shut down')
click.echo('Mandelbrot-creator shut down')
###################
# Plugins command #
###################
@cli.command()
@click.argument("number")
def start_workers(number):
"""Start the workers
"""
url = API_URL + '/start-worker/' + str(number)
try:
r = requests.get(url)
click.echo(r.text)
except requests.ConnectionError:
logging.error('Connection error in function start_workers')
click.echo('ERROR: Could not connect to the mandelbrot-creator application.\n \
Are you sure the application is started?')
except requests.Timeout:
logging.error('Timeout error in function start_workers')
click.echo('Worker took to long to respond, please try the command again.')
@cli.command()
@click.argument("number")
def spawn_workers(number):
url = API_URL + "/spawn-workers/" + str(number)
try:
r = requests.get(url)
click.echo(r.text)
except requests.ConnectionError:
logging.error('Connection error in function spawn_workers')
click.echo('ERROR: Could not connect to the mandelbrot-creator application.\n \
Are you sure the application is started?')
except requests.Timeout:
logging.error('Timeout error in function spawn_workers')
click.echo('Worker took to long to respond, please try the command again.')
click
requests
docker
docker-compose
from setuptools import setup
setup(
name='mandelbrot-creator',
version='1.0',
py_modules=['mandelbrot-creator'],
install_requires=[
'Click', 'requests', 'docker', 'docker-compose'
],
entry_points='''
[console_scripts]
mandelbrot-creator=mandelbrot_creator:cli
'''
)
version: "3.5"
services:
api:
container_name: api
build:
context: ./API
dockerfile: Dockerfile
ports:
- "8080:80"
volumes:
- "./API/:/usr/api/"
- "/var/run/docker.sock:/var/run/docker.sock"
networks:
- mandelbrot
networks:
mandelbrot:
FROM python:3.9
#######################################
# Install ping and curl for debugging #
#######################################
RUN apt-get update
# install ping
RUN apt-get -y install iputils-ping curl
###############################
# Install Python dependencies #
###############################
# Add requirements file
ADD ./requirements.txt /tmp
WORKDIR /tmp
# Upgrade pip
RUN pip install --upgrade pip
# install requirements
RUN pip install -r requirements.txt
######################
# Start microservice #
######################
WORKDIR /usr/worker
COPY . .
# Expose Port 80 for development
EXPOSE 80
CMD ["python", "./api.py"]
# Only for debugging
# CMD /bin/bash
__author__ = 'Jerico Moeyersons'
import logging
import os
import sys
import socket
from kernel import Kernel
from flask import Flask, request, jsonify
import validation
import exceptions
app = Flask(__name__)
logging.basicConfig(filename='worker.log', level=logging.WARNING)
logging.captureWarnings(True)
worker_log = logging.getLogger()
worker_log.setLevel(logging.DEBUG)
ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
worker_log.addHandler(ch)
logging.info('Worker Service started on host ' + socket.gethostname())
# Init Kernel
kernel = Kernel()
#######
# API #
#######
@app.route('/')
def info():
return 'This is the worker service'
@app.route('/start', methods=['POST'])
def start_worker():
# POST handler
if request.method == "POST":
# Parse payload
payload = request.get_json()
validation.val_start_worker(payload)
return kernel.start()
######################
# TEARDOWN FUNCTIONS #
######################
# These functions are called when the application needs to shut down
def shut_down_server():
"""Shuts down the Werkzeug simple server
Raises:
RuntimeError -- raised if the werkzeug server is not used.
"""
func = request.environ.get('werkzeug.server.shutdown')
if func is None:
raise RuntimeError('Not running with the Werkzeug Server')
func()
@app.route('/shutdown', methods=['POST'])
def shutdown():
"""Shuts down the server
Returns:
string -- notify that server is shutting down.
"""
shut_down_server()
return 'Server shutting down...'
def main():
app.run(host='0.0.0.0', port=80)
if __name__ == "__main__":
main()
__author__ = 'Jerico Moeyersons'
import logging
import http
class InvalidPayloadError(Exception):
"""Invalid payload exception
Indicates that the payload received by the microservice was invalid.
See message for more details
Inherits from exception:
Exception
"""
# HTTP status code
status_code = http.HTTPStatus.BAD_REQUEST.value
def __init__(self, message=None, status_code=None, payload=None, validation_message=None):
"""Constructor
Arguments:
message {string} -- Message for the exception
Keyword Arguments:
status_code {integer} -- HTTP status code (default: {None})
payload {object} -- Response payload (default: {None})
"""
Exception.__init__(self)
logging.warn('InvalidPayloadError: ' + message + '\n' + str(payload))
self.message = message
self.payload = payload
if status_code:
self.status_code = status_code
else:
self.status_code = InvalidPayloadError.status_code
self.validation_message = validation_message
def to_dict(self):
rv = {}
rv['received-payload'] = self.payload or ()
rv['message'] = self.message
rv['vaidation_message'] = self.validation_message
return rv
......@@ -3,6 +3,7 @@ __author__ = 'Jerico Moeyersons'
import sys
import math
import numpy as np
import http
from PIL import Image
from itertools import repeat
from multiprocessing import Pool
......@@ -30,6 +31,11 @@ class Kernel(object):
mandelbrot.save(f'{name}.png')
print(f'saved {name}.png')
message = 'Mandelbrot calculated and saved succesfully!'
code = http.HTTPStatus.OK.value
return (message, code)
def get_col(self, args):
iy, width, height, max_iterations = args
......
numpy
pillows
\ No newline at end of file
pillow
flask
jsonschema
\ No newline at end of file
__author__ = 'Jerico Moeyersons'
from jsonschema import validate
from jsonschema import ValidationError
################
# JSON schemas #
################
start_worker_schema = {
"$schema": "http://json-schema.org/draft-06/schema#",
"title": "Start Worker Schema",
"description": "The specifications to start a worker",
"type": "object",
"properties": {
"i_height": {
"description": "Height of the image to calculate",
"type": "integer"
},
"proc": {
"description": "Number of processes to calculate the mandelbrot with",
"type": "integer"
},
"max_iter": {
"description": "Maximum iterations to calculate the mandelbrot",
"type": "integer"
}
},
"required": []
}
########################
# Validation functions #
########################
def val_start_worker(payload):
"""Validate received payload against state json schema
Arguments:
payload {dict} -- payload in json format
Raises:
exceptions.InvalidPayloadError -- raises error if payload was invalid
"""
try: