diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6b46b33713c380d60b97395c30ec6bc596724032..6f2f5700f938c6e498e23ce173a31e3216f7eec1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -23,7 +23,7 @@ ci_backend: coverage: stage: coverage script: - - ./coverage.sh + - ./coverage_ci.sh - cat coverage.log artifacts: paths: diff --git a/README.md b/README.md index 1177a08d7574aa670c884cf5242b0c7bb34e22cd..0f3fcade95270c59d0e7db53515849b57aa98ae7 100644 --- a/README.md +++ b/README.md @@ -27,8 +27,10 @@ Alternatively, you can use `ssh root@localhost -p 8022 -o "UserKnownHostsFile /d To snapshot a running container, use `docker commit CONTAINER_ID`. It returns a snapshot ID, which you can access via `docker run -it SNAPSHOT_ID`. ----------------------------------------------------------------- - ## Documentation ### Changelog To update the changelog, use: `git log --oneline --decorate > CHANGELOG` + +## Testing +### Locally +To test your code locally, run `./coverage_local.sh` diff --git a/coverage.sh b/coverage_ci.sh similarity index 100% rename from coverage.sh rename to coverage_ci.sh diff --git a/coverage_local.sh b/coverage_local.sh new file mode 100755 index 0000000000000000000000000000000000000000..ae86b1287c63edaafcdae3ccdbdd61b8edd8f6ad --- /dev/null +++ b/coverage_local.sh @@ -0,0 +1,5 @@ +docker-compose build +docker-compose run --rm --entrypoint="npm run test-ci" mc_frontend > ci_frontend.log +docker-compose run --rm mcserver bash -c "source ../venv/bin/activate && coverage run --rcfile=.coveragerc tests.py && coverage combine && coverage report -m" > ci_backend.log +./coverage_ci.sh +cat coverage.log diff --git a/docker-compose.yml b/docker-compose.yml index 754af7dec1c7c174808667b93d2162db14f864cc..f42eb4ebf7ef63d298c3f029597b1da489167b10 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,18 +35,6 @@ services: tty: true volumes: - $PWD/mc_frontend/www:/home/mc/mc_frontend/www - nginx: - command: nginx -g "daemon off;" - image: nginx:alpine - ports: - - "8100:80" - restart: always - volumes: - - $PWD/mc_frontend/www:/usr/share/nginx/html - - ./mc_frontend/nginx.conf:/etc/nginx/nginx.conf - depends_on: - - mc_frontend - - mcserver mcserver: build: context: ./mc_backend @@ -63,5 +51,14 @@ services: - "5000:5000" restart: always stdin_open: true + nginx: + command: nginx -g "daemon off;" + image: nginx:alpine + ports: + - "8100:80" + restart: always + volumes: + - $PWD/mc_frontend/www:/usr/share/nginx/html + - ./mc_frontend/nginx.conf:/etc/nginx/nginx.conf volumes: db-data: diff --git a/mc_backend/.coveragerc b/mc_backend/.coveragerc index 7f0592d1bc98a983ddc9efda9f8a1cca0cbbda8f..88a16b08863b062749c2cf67a3708e429fcd8e3e 100644 --- a/mc_backend/.coveragerc +++ b/mc_backend/.coveragerc @@ -13,5 +13,6 @@ parallel = True exclude_lines = # Don't complain if non-runnable code isn't run: if __name__ == .__main__.: + except ModuleNotFoundError: ignore_errors = True show_missing = True diff --git a/mc_backend/app.py b/mc_backend/app.py index 187b386e243f1602a05d8ccf1b1338803c5a75c0..ddd570f3409b4d5f3b713da267ea06535aef3e74 100644 --- a/mc_backend/app.py +++ b/mc_backend/app.py @@ -4,4 +4,4 @@ from mcserver import get_app, get_cfg app: Flask = get_app() if __name__ == "__main__": - app.run(host=get_cfg().HOST_IP, port=get_cfg().HOST_PORT, use_reloader=False) + app.run(host=get_cfg().HOST_IP_MCSERVER, port=get_cfg().HOST_PORT, use_reloader=False) diff --git a/mc_backend/csm/__init__.py b/mc_backend/csm/__init__.py index 5070752a6cbb068eb1e068da8283fe968f23e8e1..d600973314472f7dbc310fbf2635d4ccc4b8a422 100644 --- a/mc_backend/csm/__init__.py +++ b/mc_backend/csm/__init__.py @@ -20,7 +20,7 @@ def get_cfg() -> Type[Config]: def run_app() -> None: cfg: Type[Config] = get_cfg() - get_app().run(host=cfg.HOST_IP, port=cfg.CORPUS_STORAGE_MANAGER_PORT, use_reloader=False) + get_app().run(host=cfg.HOST_IP_CSM, port=cfg.CORPUS_STORAGE_MANAGER_PORT, use_reloader=False) if __name__ == "__main__": diff --git a/mc_backend/csm/__main__.py b/mc_backend/csm/__main__.py index 38b58301b2d174ca9e2acb7504ac64d47018c119..f1ab8bdd2d142a3936c55cca8f0bdff69b60b3b4 100644 --- a/mc_backend/csm/__main__.py +++ b/mc_backend/csm/__main__.py @@ -1,3 +1,3 @@ from csm import get_app, get_cfg -get_app().run(host=get_cfg().HOST_IP, port=get_cfg().HOST_PORT, use_reloader=False) +get_app().run(host=get_cfg().HOST_IP_CSM, port=get_cfg().HOST_PORT, use_reloader=False) diff --git a/mc_backend/csm/app/api/annisFindAPI.py b/mc_backend/csm/app/api/annisFindAPI.py index 93d134bef74d5741e9eaa1d1fd9d7e27a4d4c06c..90d96af0caa2a34d19599060546e1bfd6ba8b935 100644 --- a/mc_backend/csm/app/api/annisFindAPI.py +++ b/mc_backend/csm/app/api/annisFindAPI.py @@ -1,11 +1,13 @@ import flask -from flask_restful import Resource, reqparse -from mcserver.app.services import NetworkService, CorpusService, AnnotationService +from flask_restful import Resource +from flask_restful.reqparse import RequestParser + +from mcserver.app.services import NetworkService, CorpusService class AnnisFindAPI(Resource): def __init__(self): - self.reqparse = reqparse.RequestParser() + self.reqparse: RequestParser = NetworkService.base_request_parser.copy() self.reqparse.add_argument("aql", type=str, required=True, location="form", help="No AQL provided") self.reqparse.add_argument("urn", type=str, required=True, default="", location="form", help="No URN provided") super(AnnisFindAPI, self).__init__() diff --git a/mc_backend/csm/app/api/corpusStorageManagerAPI.py b/mc_backend/csm/app/api/corpusStorageManagerAPI.py index 44523f4fc766d506af4636c2b45a958585b205a9..3a24a61db4cc97bff6d41c291049fac00a86f30b 100644 --- a/mc_backend/csm/app/api/corpusStorageManagerAPI.py +++ b/mc_backend/csm/app/api/corpusStorageManagerAPI.py @@ -4,7 +4,8 @@ from typing import Dict, List import flask from conllu import TokenList -from flask_restful import Resource, reqparse, abort +from flask_restful import Resource, abort +from flask_restful.reqparse import RequestParser from mcserver.app.models import ExerciseType, Phenomenon, AnnisResponse from mcserver.app.services import CorpusService, NetworkService @@ -16,7 +17,7 @@ class CorpusStorageManagerAPI(Resource): It manages the database and everything corpus-related.""" def __init__(self): - self.reqparse = reqparse.RequestParser() + self.reqparse: RequestParser = NetworkService.base_request_parser.copy() self.reqparse.add_argument("title", type=str, required=True, location="data", help="No title provided") self.reqparse.add_argument("annotations", required=True, location="data", help="No annotations provided") diff --git a/mc_backend/csm/app/api/frequencyAPI.py b/mc_backend/csm/app/api/frequencyAPI.py index bfbb03fe38aac5a623f0e55a6dd10c00873b8bbb..070e14305856973baab0ff626009d22a9e0d6cf5 100644 --- a/mc_backend/csm/app/api/frequencyAPI.py +++ b/mc_backend/csm/app/api/frequencyAPI.py @@ -1,13 +1,15 @@ from typing import List, Dict, Set import flask -from flask_restful import Resource, reqparse +from flask_restful import Resource +from flask_restful.reqparse import RequestParser + from mcserver.app.models import FrequencyAnalysis, Phenomenon from mcserver.app.services import NetworkService, CorpusService, AnnotationService class FrequencyAPI(Resource): def __init__(self): - self.reqparse = reqparse.RequestParser() + self.reqparse: RequestParser = NetworkService.base_request_parser.copy() self.reqparse.add_argument("urn", type=str, required=True, default="", location="form", help="No URN provided") super(FrequencyAPI, self).__init__() diff --git a/mc_backend/csm/app/api/subgraphAPI.py b/mc_backend/csm/app/api/subgraphAPI.py index fe0211d081f117bb2fc0c4371f135f8ce2c0e330..0c3be6cdef3df7824c94de83e7f2c1ac4eace98a 100644 --- a/mc_backend/csm/app/api/subgraphAPI.py +++ b/mc_backend/csm/app/api/subgraphAPI.py @@ -2,7 +2,9 @@ import json from typing import Dict, List import flask -from flask_restful import Resource, reqparse +from flask_restful import Resource +from flask_restful.reqparse import RequestParser + from mcserver.app.models import ExerciseData, GraphData, Solution, SolutionElement, AnnisResponse from mcserver.app.services import CorpusService, AnnotationService, NetworkService @@ -10,7 +12,7 @@ from mcserver.app.services import CorpusService, AnnotationService, NetworkServi class SubgraphAPI(Resource): def __init__(self): - self.reqparse = reqparse.RequestParser() + self.reqparse: RequestParser = NetworkService.base_request_parser.copy() self.reqparse.add_argument("aqls", required=False, location="data", help="No AQLs provided", action="append") self.reqparse.add_argument("ctx_left", type=str, required=False, default="", location="data", help="No left context provided") diff --git a/mc_backend/csm/app/api/textcomplexityAPI.py b/mc_backend/csm/app/api/textcomplexityAPI.py index 5b3bd910bd93e2528bf80fac12718ac14706c491..021fa52f2f032dd0d4223aed044a338625fefb74 100644 --- a/mc_backend/csm/app/api/textcomplexityAPI.py +++ b/mc_backend/csm/app/api/textcomplexityAPI.py @@ -1,7 +1,8 @@ import rapidjson as json import flask -from flask_restful import Resource, reqparse +from flask_restful import Resource +from flask_restful.reqparse import RequestParser from mcserver.app.models import AnnisResponse, GraphData, TextComplexity from mcserver.app.services import NetworkService, CorpusService, TextComplexityService @@ -11,7 +12,7 @@ class TextComplexityAPI(Resource): """The Text Complexity API resource. It gives users measures for text complexity for a given text.""" def __init__(self): - self.reqparse = reqparse.RequestParser() + self.reqparse: RequestParser = NetworkService.base_request_parser.copy() self.reqparse.add_argument('urn', type=str, location="data", required=True, help='No URN provided') self.reqparse.add_argument('measure', type=str, location="data", required=True, help='No MEASURE provided') self.reqparse.add_argument('annis_response', type=dict, location="data", required=False, diff --git a/mc_backend/csm/gunicorn_config.py b/mc_backend/csm/gunicorn_config.py index c378e3db2f75bc7a23558ef0d42344fe8f0afa86..31f377d6f3e4701dadbb308e529ec110277165cc 100644 --- a/mc_backend/csm/gunicorn_config.py +++ b/mc_backend/csm/gunicorn_config.py @@ -1,7 +1,7 @@ """Configuration for the gunicorn server""" from mcserver import Config -bind = "{0}:{1}".format(Config.HOST_IP, Config.CORPUS_STORAGE_MANAGER_PORT) +bind = "{0}:{1}".format(Config.HOST_IP_CSM, Config.CORPUS_STORAGE_MANAGER_PORT) debug = False reload = True timeout = 3600 diff --git a/mc_backend/mcserver/__init__.py b/mc_backend/mcserver/__init__.py index 9ac1fe86c0839cca72a9cb837413c20583fccbe2..25828bb6ebbbd3bd0c41f2164392b9cd5524a898 100644 --- a/mc_backend/mcserver/__init__.py +++ b/mc_backend/mcserver/__init__.py @@ -20,4 +20,4 @@ def get_cfg() -> Type[Config]: if __name__ == "__main__": # reloader has to be disabled because of a bug with Flask and multiprocessing - get_app().run(host=get_cfg().HOST_IP, port=get_cfg().HOST_PORT, use_reloader=False) + get_app().run(host=get_cfg().HOST_IP_MCSERVER, port=get_cfg().HOST_PORT, use_reloader=False) diff --git a/mc_backend/mcserver/__main__.py b/mc_backend/mcserver/__main__.py index 03c38620b456af9c9aecce991cccfbabf8948f0e..2a07afee458805480e0f6f81a644cb106901ee35 100644 --- a/mc_backend/mcserver/__main__.py +++ b/mc_backend/mcserver/__main__.py @@ -1,3 +1,3 @@ from mcserver import get_app, get_cfg -get_app().run(host=get_cfg().HOST_IP, port=get_cfg().HOST_PORT, use_reloader=False) +get_app().run(host=get_cfg().HOST_IP_MCSERVER, port=get_cfg().HOST_PORT, use_reloader=False) diff --git a/mc_backend/mcserver/app/__init__.py b/mc_backend/mcserver/app/__init__.py index 016aefda259f91d31efea6701c5983a86f817f39..56c38cf8e5a8786f7951e66bb906bc68afa76de0 100644 --- a/mc_backend/mcserver/app/__init__.py +++ b/mc_backend/mcserver/app/__init__.py @@ -5,7 +5,9 @@ from logging.handlers import RotatingFileHandler from threading import Thread from time import strftime from typing import Type -from flask import Flask, got_request_exception, request, Response + +import flask +from flask import Flask, got_request_exception, request, Response, send_from_directory from flask_cors import CORS from flask_migrate import Migrate from flask_sqlalchemy import SQLAlchemy @@ -59,6 +61,12 @@ def init_app_common(cfg: Type[Config] = Config, is_csm: bool = False) -> Flask: request.full_path, response.status) return response + @app.route(Config.SERVER_URI_FAVICON) + def get_favicon(): + """Sends the favicon to browsers, which is used, e.g., in the tabs as a symbol for our application.""" + mime_type: str = 'image/vnd.microsoft.icon' + return send_from_directory(Config.ASSETS_DIRECTORY, Config.FAVICON_FILE_NAME, mimetype=mime_type) + @app.teardown_appcontext def shutdown_session(exception=None): """ Shuts down the session when the application exits. (maybe also after every request ???) """ @@ -100,7 +108,7 @@ def log_exception(sender_app: Flask, exception, **extra): **extra -- any additional arguments """ # TODO: RETURN ERROR IN JSON FORMAT - sender_app.logger.exception("ERROR") + sender_app.logger.exception(f"ERROR for {flask.request.url}") def start_updater(app: Flask) -> Thread: diff --git a/mc_backend/mcserver/app/api/corpusAPI.py b/mc_backend/mcserver/app/api/corpusAPI.py index e3d6bd1c1da3f4f165fec1459a6d52726e7abd64..31f4dd488cf1c7e429186e2b483c1c3cf5084b82 100644 --- a/mc_backend/mcserver/app/api/corpusAPI.py +++ b/mc_backend/mcserver/app/api/corpusAPI.py @@ -1,8 +1,10 @@ """The corpus API. Add it to your REST API to provide users with metadata about specific texts.""" -from flask_restful import Resource, reqparse, abort, marshal +from flask_restful import Resource, abort, marshal +from flask_restful.reqparse import RequestParser from mcserver.app import db from mcserver.app.models import Corpus, corpus_fields +from mcserver.app.services import NetworkService class CorpusAPI(Resource): @@ -10,7 +12,7 @@ class CorpusAPI(Resource): def __init__(self): """Initialize possible arguments for calls to the corpus REST API.""" - self.reqparse = reqparse.RequestParser() + self.reqparse: RequestParser = NetworkService.base_request_parser.copy() self.reqparse.add_argument("title", type=str, required=False, help="No title provided") self.reqparse.add_argument("author", type=str, required=False, help="No author provided") self.reqparse.add_argument("source_urn", type=str, required=False, help="No source URN provided") diff --git a/mc_backend/mcserver/app/api/corpusListAPI.py b/mc_backend/mcserver/app/api/corpusListAPI.py index 77b02c77e5fa29a7a244f09c40bc9dd63c62c4ef..fc6cb513c1ce1e9d9097b711670e9ee99e061eb3 100644 --- a/mc_backend/mcserver/app/api/corpusListAPI.py +++ b/mc_backend/mcserver/app/api/corpusListAPI.py @@ -2,12 +2,13 @@ from datetime import datetime from flask import jsonify -from flask_restful import Resource, reqparse, marshal +from flask_restful import Resource, marshal +from flask_restful.reqparse import RequestParser from sqlalchemy.exc import OperationalError, InvalidRequestError from mcserver.app import db from mcserver.app.models import UpdateInfo, ResourceType, Corpus, corpus_fields -from mcserver.app.services import CorpusService +from mcserver.app.services import CorpusService, NetworkService class CorpusListAPI(Resource): @@ -15,7 +16,7 @@ class CorpusListAPI(Resource): def __init__(self): """Initialize possible arguments for calls to the corpus list REST API.""" - self.reqparse = reqparse.RequestParser() + self.reqparse: RequestParser = NetworkService.base_request_parser.copy() self.reqparse.add_argument("last_update_time", type=int, required=True, help="No milliseconds time for last update provided") super(CorpusListAPI, self).__init__() diff --git a/mc_backend/mcserver/app/api/exerciseAPI.py b/mc_backend/mcserver/app/api/exerciseAPI.py index e2cf3453b4bb1e68766d27f7a012d791a476fc3d..7b9249fd074768546b5c74d6a6e0b461a80f7979 100644 --- a/mc_backend/mcserver/app/api/exerciseAPI.py +++ b/mc_backend/mcserver/app/api/exerciseAPI.py @@ -6,7 +6,8 @@ import rapidjson as json from typing import List, Dict import requests -from flask_restful import Resource, reqparse, marshal, abort +from flask_restful import Resource, marshal, abort +from flask_restful.reqparse import RequestParser from mcserver.app import db from mcserver.app.models import ExerciseType, Solution, ExerciseData, Exercise, exercise_fields, AnnisResponse, \ @@ -21,7 +22,7 @@ class ExerciseAPI(Resource): def __init__(self): """Initialize possible arguments for calls to the exercise REST API.""" # TODO: switch to other request parser, e.g. Marshmallow, because the one used by Flask-RESTful does not allow parsing arguments from different locations, e.g. one argument from 'location=args' and another argument from 'location=form' - self.reqparse = reqparse.RequestParser() + self.reqparse: RequestParser = NetworkService.base_request_parser.copy() self.reqparse.add_argument("urn", type=str, required=False, location="form", help="No URN provided") self.reqparse.add_argument("type", type=str, required=False, location="form", help="No exercise type provided") self.reqparse.add_argument("search_values", type=str, required=False, location="form", @@ -49,6 +50,8 @@ class ExerciseAPI(Resource): ar: AnnisResponse = CorpusService.get_corpus(cts_urn=exercise.urn, is_csm=False) if not ar.nodes: abort(404) + exercise.last_access_time = datetime.utcnow() + db.session.commit() exercise_type: ExerciseType = ExerciseType(exercise.exercise_type) ar.solutions = json.loads(exercise.solutions) ar.uri = exercise.uri diff --git a/mc_backend/mcserver/app/api/exerciseListAPI.py b/mc_backend/mcserver/app/api/exerciseListAPI.py index 26e5d79cc02ca9e66b2149b357ca4ab6b43766c7..8ff5c4df18828e9bf88cbc5b15a66c18f1ed1eaf 100644 --- a/mc_backend/mcserver/app/api/exerciseListAPI.py +++ b/mc_backend/mcserver/app/api/exerciseListAPI.py @@ -4,7 +4,9 @@ from typing import List, Set import conllu from conllu import TokenList -from flask_restful import Resource, reqparse +from flask_restful import Resource +from flask_restful.reqparse import RequestParser + from mcserver.app.models import Exercise, Language, VocabularyCorpus, UpdateInfo, ResourceType from mcserver.app.services import NetworkService, FileService @@ -14,7 +16,7 @@ class ExerciseListAPI(Resource): def __init__(self): """Initialize possible arguments for calls to the exercise list REST API.""" - self.reqparse = reqparse.RequestParser() + self.reqparse: RequestParser = NetworkService.base_request_parser.copy() self.reqparse.add_argument("lang", type=str, required=True, help="No language specified") self.reqparse.add_argument("last_update_time", type=int, required=False, default=0, help="No milliseconds time for last update provided") diff --git a/mc_backend/mcserver/app/api/fileAPI.py b/mc_backend/mcserver/app/api/fileAPI.py index faea453a4be832a193a392d84260a163b4f1a583..d12af5a40c3bbf28194fa639184c396f854647b5 100644 --- a/mc_backend/mcserver/app/api/fileAPI.py +++ b/mc_backend/mcserver/app/api/fileAPI.py @@ -7,7 +7,8 @@ from typing import List, Union import flask from flask import send_from_directory -from flask_restful import Resource, reqparse, abort +from flask_restful import Resource, abort +from flask_restful.reqparse import RequestParser from werkzeug.wrappers import ETagResponseMixin from mcserver.app import db @@ -22,7 +23,7 @@ class FileAPI(Resource): def __init__(self): """Initialize possible arguments for calls to the file REST API.""" - self.reqparse = reqparse.RequestParser() + self.reqparse: RequestParser = NetworkService.base_request_parser.copy() self.reqparse.add_argument("id", type=str, required=False, location="args", help="No exercise ID or URN provided") self.reqparse.add_argument("type", type=str, required=False, location="args", help="No file type provided") diff --git a/mc_backend/mcserver/app/api/frequencyAPI.py b/mc_backend/mcserver/app/api/frequencyAPI.py index f82b38ac48165d13950ef750fd8f560bfef68598..fb3ce42175c023af3fd2d325bf8ef6f4f41dc9c5 100644 --- a/mc_backend/mcserver/app/api/frequencyAPI.py +++ b/mc_backend/mcserver/app/api/frequencyAPI.py @@ -1,7 +1,8 @@ import flask import requests -from flask_restful import Resource, reqparse +from flask_restful import Resource import rapidjson as json +from flask_restful.reqparse import RequestParser from mcserver import Config from mcserver.app.services import NetworkService @@ -10,7 +11,7 @@ from mcserver.app.services import NetworkService class FrequencyAPI(Resource): def __init__(self): # TODO: FIX THE REQUEST PARSING FOR ALL APIs - self.reqparse = reqparse.RequestParser() + self.reqparse: RequestParser = NetworkService.base_request_parser.copy() self.reqparse.add_argument("urn", type=str, required=True, default="", location="form", help="No URN provided") super(FrequencyAPI, self).__init__() diff --git a/mc_backend/mcserver/app/api/h5pAPI.py b/mc_backend/mcserver/app/api/h5pAPI.py index de0cb2853d310e89869eb7e20466c9f417998062..efc12c94455d783bf4ab8c8ed49813e5afa5703f 100644 --- a/mc_backend/mcserver/app/api/h5pAPI.py +++ b/mc_backend/mcserver/app/api/h5pAPI.py @@ -1,7 +1,8 @@ import json from typing import List -from flask_restful import Resource, reqparse, abort +from flask_restful import Resource, abort +from flask_restful.reqparse import RequestParser from mcserver.app.models import Language, Exercise, ExerciseType, Solution from mcserver.app.services import TextService, NetworkService @@ -12,7 +13,7 @@ class H5pAPI(Resource): def __init__(self): """Initialize possible arguments for calls to the H5P REST API.""" - self.reqparse = reqparse.RequestParser() + self.reqparse: RequestParser = NetworkService.base_request_parser.copy() self.reqparse.add_argument("eid", type=str, required=True, default="", help="No exercise ID provided") self.reqparse.add_argument("lang", type=str, required=True, default="en", help="No language code provided") self.reqparse.add_argument("solution_indices", type=str, required=False, help="No solution IDs provided") diff --git a/mc_backend/mcserver/app/api/kwicAPI.py b/mc_backend/mcserver/app/api/kwicAPI.py index d850f45265df8570a3bd2b163f716a86f23af854..88eb85902950ea5b1ef1e2f7111e5c3381457440 100644 --- a/mc_backend/mcserver/app/api/kwicAPI.py +++ b/mc_backend/mcserver/app/api/kwicAPI.py @@ -10,7 +10,8 @@ from typing import List, Dict import requests from bs4 import BeautifulSoup, ResultSet, Tag from conllu import TokenList -from flask_restful import Resource, reqparse +from flask_restful import Resource +from flask_restful.reqparse import RequestParser from mcserver.app.models import ExerciseType, ExerciseData, LinkMC, NodeMC from mcserver.app.services import AnnotationService, NetworkService @@ -22,7 +23,7 @@ class KwicAPI(Resource): def __init__(self): """Initializes possible arguments for calls to the KWIC REST API.""" - self.reqparse = reqparse.RequestParser() + self.reqparse: RequestParser = NetworkService.base_request_parser.copy() self.reqparse.add_argument("urn", type=str, required=True, default="", location="form", help="No URN provided") self.reqparse.add_argument("search_values", type=str, required=True, location="form", help="No search value(s) provided") diff --git a/mc_backend/mcserver/app/api/rawTextAPI.py b/mc_backend/mcserver/app/api/rawTextAPI.py index 797cd15366b033335a46a4c38a809399f5b2aadf..5ebf6f1aea8bf14b3e88cfadfdad5fb8e0ad137e 100644 --- a/mc_backend/mcserver/app/api/rawTextAPI.py +++ b/mc_backend/mcserver/app/api/rawTextAPI.py @@ -1,4 +1,6 @@ -from flask_restful import Resource, reqparse, abort +from flask_restful import Resource, abort +from flask_restful.reqparse import RequestParser + from mcserver.app.models import AnnisResponse, TextComplexityMeasure, GraphData from mcserver.app.services import CorpusService, NetworkService, TextComplexityService @@ -8,7 +10,7 @@ class RawTextAPI(Resource): def __init__(self): """Initialize possible arguments for calls to the fill the blank REST API.""" - self.reqparse = reqparse.RequestParser() + self.reqparse: RequestParser = NetworkService.base_request_parser.copy() self.reqparse.add_argument("urn", type=str, required=True, default="", help="No URN provided") super(RawTextAPI, self).__init__() diff --git a/mc_backend/mcserver/app/api/staticExercisesAPI.py b/mc_backend/mcserver/app/api/staticExercisesAPI.py index f2dced5aa85f6c68b1200f634e64f5405f6561f9..524b6ec6704f236ee83a813d0575371da6947516 100644 --- a/mc_backend/mcserver/app/api/staticExercisesAPI.py +++ b/mc_backend/mcserver/app/api/staticExercisesAPI.py @@ -11,7 +11,8 @@ from typing import Dict, List, Set, Match, Tuple from zipfile import ZipFile import requests -from flask_restful import Resource, reqparse, abort +from flask_restful import Resource, abort +from flask_restful.reqparse import RequestParser from requests import Response from mcserver.app.models import StaticExercise @@ -24,7 +25,7 @@ class StaticExercisesAPI(Resource): def __init__(self): """Initialize possible arguments for calls to the StaticExercises REST API.""" - self.reqparse = reqparse.RequestParser() + self.reqparse: RequestParser = NetworkService.base_request_parser.copy() super(StaticExercisesAPI, self).__init__() def get(self): diff --git a/mc_backend/mcserver/app/api/textcomplexityAPI.py b/mc_backend/mcserver/app/api/textcomplexityAPI.py index 15929cd5b983f98d98492a6439f769791b6fc6e1..e0ee427d83fda309a351ac35afaab73c4bb887a2 100644 --- a/mc_backend/mcserver/app/api/textcomplexityAPI.py +++ b/mc_backend/mcserver/app/api/textcomplexityAPI.py @@ -1,4 +1,6 @@ -from flask_restful import Resource, reqparse +from flask_restful import Resource +from flask_restful.reqparse import RequestParser + from mcserver.app.models import AnnisResponse, GraphData, TextComplexity from mcserver.app.services import NetworkService, CorpusService, TextComplexityService @@ -7,7 +9,7 @@ class TextComplexityAPI(Resource): """The Text Complexity API resource. It gives users measures for text complexity for a given text.""" def __init__(self): - self.reqparse = reqparse.RequestParser() + self.reqparse: RequestParser = NetworkService.base_request_parser.copy() self.reqparse.add_argument('urn', type=str, required=True, help='No URN provided') self.reqparse.add_argument('measure', type=str, required=True, help='No MEASURE provided') super(TextComplexityAPI, self).__init__() diff --git a/mc_backend/mcserver/app/api/validReffAPI.py b/mc_backend/mcserver/app/api/validReffAPI.py index ab32febce3648340f86c2a5bdec24ea8e3037813..ee4ca99f409ff11e81dfb060531adf41e0274ee5 100644 --- a/mc_backend/mcserver/app/api/validReffAPI.py +++ b/mc_backend/mcserver/app/api/validReffAPI.py @@ -1,5 +1,7 @@ from typing import List -from flask_restful import Resource, reqparse, abort +from flask_restful import Resource, abort +from flask_restful.reqparse import RequestParser + from mcserver.app.services import CorpusService, NetworkService, CustomCorpusService @@ -8,7 +10,7 @@ class ValidReffAPI(Resource): def __init__(self): """Initialize possible arguments for calls to the valid references REST API.""" - self.reqparse = reqparse.RequestParser() + self.reqparse: RequestParser = NetworkService.base_request_parser.copy() self.reqparse.add_argument("urn", type=str, required=True, default="", help="No URN provided") super(ValidReffAPI, self).__init__() diff --git a/mc_backend/mcserver/app/api/vectorNetworkAPI.py b/mc_backend/mcserver/app/api/vectorNetworkAPI.py index bf7e00633fd8799a0219d156bb37fb128c22c066..cec07320522814ed7d74b42886f2f544d8ca5c61 100644 --- a/mc_backend/mcserver/app/api/vectorNetworkAPI.py +++ b/mc_backend/mcserver/app/api/vectorNetworkAPI.py @@ -2,7 +2,8 @@ import os import re from typing import List, Dict, Set, Tuple, Pattern -from flask_restful import Resource, reqparse +from flask_restful import Resource +from flask_restful.reqparse import RequestParser from gensim import matutils from gensim.models import Word2Vec from matplotlib import pyplot @@ -18,7 +19,7 @@ class VectorNetworkAPI(Resource): def __init__(self): """Initialize possible arguments for calls to the corpus list REST API.""" - self.reqparse = reqparse.RequestParser() + self.reqparse: RequestParser = NetworkService.base_request_parser.copy() self.reqparse.add_argument("search_regex", type=str, required=True, help="No regular expression provided for the search") self.reqparse.add_argument("min_count", type=int, required=False, default=1, diff --git a/mc_backend/mcserver/app/api/vocabularyAPI.py b/mc_backend/mcserver/app/api/vocabularyAPI.py index cd77c6e4da0275d83d166f9a61249e9a5737f7be..6aa076a95511afa676169e4605305fe6536f3d9c 100644 --- a/mc_backend/mcserver/app/api/vocabularyAPI.py +++ b/mc_backend/mcserver/app/api/vocabularyAPI.py @@ -1,6 +1,8 @@ import string from typing import Set, List, Dict -from flask_restful import Resource, reqparse, inputs +from flask_restful import Resource, inputs +from flask_restful.reqparse import RequestParser + from mcserver.app.models import VocabularyCorpus, GraphData, Sentence, AnnisResponse, TextComplexityMeasure from mcserver.app.services import FileService, CorpusService, AnnotationService, NetworkService, TextService, \ TextComplexityService @@ -12,7 +14,7 @@ class VocabularyAPI(Resource): It shows whether the vocabulary of a text matches that of another one.""" def __init__(self): - self.reqparse = reqparse.RequestParser() + self.reqparse: RequestParser = NetworkService.base_request_parser.copy() self.reqparse.add_argument("query_urn", type=str, required=True, help="No URN for the query corpus provided") self.reqparse.add_argument("vocabulary", type=str, required=True, help="No reference vocabulary provided") self.reqparse.add_argument("frequency_upper_bound", type=int, required=True, diff --git a/mc_backend/mcserver/app/assets/favicon.ico b/mc_backend/mcserver/app/assets/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..5d2411be4e09f1e693698f66e23efe9165dec430 Binary files /dev/null and b/mc_backend/mcserver/app/assets/favicon.ico differ diff --git a/mc_backend/mcserver/app/services/networkService.py b/mc_backend/mcserver/app/services/networkService.py index 74abaf6a7c62bc0ddf880fc9a19a642ca3843ba0..b26a40d81541b326bd2ec3542e5a25307d5414ed 100644 --- a/mc_backend/mcserver/app/services/networkService.py +++ b/mc_backend/mcserver/app/services/networkService.py @@ -4,11 +4,13 @@ from typing import Dict import rapidjson from flask import Response +from flask_restful.reqparse import RequestParser from mcserver.app.models import StaticExercise class NetworkService: + base_request_parser: RequestParser = RequestParser(bundle_errors=True) exercises: Dict[str, StaticExercise] = {} exercises_last_update: datetime = datetime.fromtimestamp(0) diff --git a/mc_backend/mcserver/config.py b/mc_backend/mcserver/config.py index 46208f0a635fe6420ac02c8e50a9060d8ad08a51..b942a2e849e43930b090689225a6a63c28d41b0b 100644 --- a/mc_backend/mcserver/config.py +++ b/mc_backend/mcserver/config.py @@ -45,7 +45,6 @@ class Config(object): COVERAGE_CONFIGURATION_FILE_NAME = ".coveragerc" COVERAGE_ENVIRONMENT_VARIABLE = "COVERAGE_PROCESS_START" CSM_DIRECTORY = os.path.join(CURRENT_WORKING_DIRECTORY, "csm") - CSM_NETWORK_DOCKER = "csm" CSRF_ENABLED = True CTS_API_BASE_URL = "https://cts.perseids.org/api/cts/" CUSTOM_CORPUS_CAES_GAL_FILE_PATH = os.path.join(TREEBANKS_PROIEL_PATH, "caes-gal.conllu") @@ -62,6 +61,9 @@ class Config(object): DATABASE_URL_FALLBACK = "postgresql://postgres@db:5432/" if IS_DOCKER else DATABASE_URL_LOCAL DATABASE_URL = os.environ.get("DATABASE_URL", DATABASE_URL_FALLBACK) DEBUG = False + DOCKER_SERVICE_NAME_CSM = "csm" + DOCKER_SERVICE_NAME_MCSERVER = "mcserver" + FAVICON_FILE_NAME = "favicon.ico" FLASK_MIGRATE = "migrate" GRAPHANNIS_DEPENDENCY_LINK = "dep" GRAPHANNIS_LOG_PATH = os.path.join(os.getcwd(), "graphannis.log") @@ -69,8 +71,8 @@ class Config(object): H5P_FILL_BLANKS = "fill_blanks" H5P_MULTI_CHOICE = "multi_choice" H5P_VOC_LIST = "voc_list" - HOST_IP = os.environ.get("HOST_IP", "0.0.0.0" if IS_DOCKER else "127.0.0.1") - HOST_IP_CSM = CSM_NETWORK_DOCKER if IS_DOCKER else HOST_IP + HOST_IP_MCSERVER = os.environ.get("HOST_IP", DOCKER_SERVICE_NAME_MCSERVER if IS_DOCKER else "127.0.0.1") + HOST_IP_CSM = DOCKER_SERVICE_NAME_CSM if IS_DOCKER else HOST_IP_MCSERVER HOST_PORT = 5000 INTERNET_PROTOCOL = "http://" INTERVAL_CORPUS_AGE_CHECK = 60 * 60 @@ -80,8 +82,8 @@ class Config(object): INTERVAL_STATIC_EXERCISES = 60 * 60 * 24 IS_PRODUCTION = os.environ.get("FLASK_ENV_VARIABLE", "development") == "production" LEARNING_ANALYTICS_DIRECTORY = os.path.join(FILES_DIRECTORY, "learning_analytics") - LOG_PATH_CSM = "csm.log" - LOG_PATH_MCSERVER = "mcserver.log" + LOG_PATH_CSM = f"{DOCKER_SERVICE_NAME_CSM}.log" + LOG_PATH_MCSERVER = f"{DOCKER_SERVICE_NAME_MCSERVER}.log" MIGRATIONS_DIRECTORY = os.path.join(MC_SERVER_DIRECTORY, "migrations") NETWORK_GRAPH_TMP_PATH = os.path.join(TMP_DIRECTORY, "graph.svg") PANEGYRICI_LATINI_DIRECTORY = os.path.join(ASSETS_DIRECTORY, "panegyrici_latini") @@ -100,6 +102,7 @@ class Config(object): SERVER_URI_CSM_SUBGRAPH = SERVER_URI_CSM + "subgraph" SERVER_URI_EXERCISE = SERVER_URI_BASE + "exercise" SERVER_URI_EXERCISE_LIST = SERVER_URI_BASE + "exerciseList" + SERVER_URI_FAVICON = "/favicon.ico" SERVER_URI_FILE = SERVER_URI_BASE + "file" SERVER_URI_FREQUENCY = SERVER_URI_BASE + "frequency" SERVER_URI_H5P = SERVER_URI_BASE + "h5p" @@ -152,9 +155,9 @@ class DevelopmentConfig(Config): class TestingConfig(Config): """Configuration for testing""" CTS_API_GET_PASSAGE_URL = "https://cts.perseids.org/api/cts/?request=GetPassage&urn=urn:cts:latinLit:phi1351.phi002.perseus-lat1:2.2" - HOST_IP = "0.0.0.0" + HOST_IP_MCSERVER = "0.0.0.0" PRESERVE_CONTEXT_ON_EXCEPTION = False - SERVER_NAME = Config.HOST_IP + ":{0}".format(Config.HOST_PORT) + SERVER_NAME = Config.HOST_IP_MCSERVER + ":{0}".format(Config.HOST_PORT) SESSION_COOKIE_DOMAIN = False SIMULATE_CORPUS_NOT_FOUND = False SIMULATE_EMPTY_GRAPH = False diff --git a/mc_backend/mcserver/gunicorn_config.py b/mc_backend/mcserver/gunicorn_config.py index 1ef14fdc2a9418df25ea0eb0f067acc190358461..7ba3f83b7825b4506d0fd8d85f1b3f98fbef3154 100644 --- a/mc_backend/mcserver/gunicorn_config.py +++ b/mc_backend/mcserver/gunicorn_config.py @@ -3,7 +3,7 @@ import multiprocessing from mcserver import Config -bind = "{0}:{1}".format(Config.HOST_IP, Config.HOST_PORT) +bind = "{0}:{1}".format(Config.HOST_IP_MCSERVER, Config.HOST_PORT) debug = False reload = True timeout = 3600 diff --git a/mc_backend/run_csm.py b/mc_backend/run_csm.py index b923a0ee71a4e0cfbedd4be3cf85449bf92c1338..6265a76d2dfb57c4338692157ff6f84e1fcbb4e4 100644 --- a/mc_backend/run_csm.py +++ b/mc_backend/run_csm.py @@ -4,4 +4,4 @@ from csm import get_app, get_cfg app: Flask = get_app() if __name__ == "__main__": - app.run(host=get_cfg().HOST_IP, port=get_cfg().CORPUS_STORAGE_MANAGER_PORT, use_reloader=False) + app.run(host=get_cfg().HOST_IP_CSM, port=get_cfg().CORPUS_STORAGE_MANAGER_PORT, use_reloader=False) diff --git a/mc_backend/tests.py b/mc_backend/tests.py index d8e7350e6d270fe8b9f541c658b88fa5df29619f..460832583fc906eb524794bf5be90ea5ba826543 100644 --- a/mc_backend/tests.py +++ b/mc_backend/tests.py @@ -1,5 +1,4 @@ """Unit tests for testing the application functionality.""" -# TODO: TEST BLINKER, GUNICORN, PSYCOPG2-BINARY AND COVERAGE IMPORTS import copy import logging import ntpath @@ -495,6 +494,14 @@ class McTestCase(unittest.TestCase): cfg.SQLALCHEMY_DATABASE_URI = old_uri self.app_context.push() + def test_get_favicon(self): + """Sends the favicon to browsers, which is used, e.g., in the tabs as a symbol for our application.""" + response: Response = self.client.get(Config.SERVER_URI_FAVICON) + with open(os.path.join(Config.ASSETS_DIRECTORY, Config.FAVICON_FILE_NAME), "rb") as f: + content: bytes = f.read() + data_received: bytes = response.get_data() + self.assertEqual(content, data_received) + def test_init_corpus_storage_manager(self): """ Initializes the corpus storage manager. """ ui_cts: UpdateInfo = UpdateInfo(resource_type=ResourceType.cts_data, last_modified_time=datetime.utcnow()) @@ -854,6 +861,18 @@ class CommonTestCase(unittest.TestCase): FileType.pdf, []) self.assertEqual(xml_string, Mocks.exercise_xml) + def test_dependency_imports(self): + """Verifies that all necessary dependencies are installed by trying to import them.""" + are_dependencies_missing = False + try: + import blinker # for signalling + import gunicorn # for production server + import psycopg2 # for database access via SQLAlchemy + import coverage # for code coverage in unit tests + except ModuleNotFoundError: + are_dependencies_missing = True + self.assertFalse(are_dependencies_missing) + def test_extract_custom_corpus_text(self): """ Extracts text from the relevant parts of a (custom) corpus. """ new_text_parts: List[Tuple[str, str]] = CustomCorpusService.extract_custom_corpus_text( diff --git a/mc_frontend/nginx.conf b/mc_frontend/nginx.conf index e0600b0f1b87f1cd7591582056f2e644c440121e..41e3accdfa4af90b0b769fe215190ecca93972cd 100644 --- a/mc_frontend/nginx.conf +++ b/mc_frontend/nginx.conf @@ -27,8 +27,6 @@ http { #charset koi8-r; - #access_log logs/host.access.log main; - location / { root /usr/share/nginx/html; index index.html index.htm; @@ -39,16 +37,10 @@ http { } location /mc-service/ { - proxy_pass http://mcserver:5000/; - } - - #error_page 404 /404.html; - - # redirect server error pages to the static page /50x.html - # - error_page 500 502 503 504 /50x.html; - location = /50x.html { - root html; + resolver 127.0.0.11 valid=30s; + set $backend_host mcserver; + rewrite /mc-service/(.*) /$1 break; + proxy_pass http://$backend_host:5000; } # proxy the PHP scripts to Apache listening on 127.0.0.1:80