added a script for testing code coverage locally

.gitlab-ci.yml

@@ -23,7 +23,7 @@ ci_backend:
 coverage:
   stage: coverage
   script:
-    - ./coverage.sh
+    - ./coverage_ci.sh
     - cat coverage.log
   artifacts:
     paths:

README.md

@@ -27,8 +27,10 @@ Alternatively, you can use `ssh root@localhost -p 8022 -o "UserKnownHostsFile /d
 
 To snapshot a running container, use `docker commit CONTAINER_ID`. It returns a snapshot ID, which you can access via `docker run -it SNAPSHOT_ID`.
 
-----------------------------------------------------------------
-
 ## Documentation
 ### Changelog
 To update the changelog, use: `git log --oneline --decorate > CHANGELOG`
+
+## Testing
+### Locally
+To test your code locally, run `./coverage_local.sh`

coverage_local.sh

+docker-compose build
+docker-compose run --rm --entrypoint="npm run test-ci" mc_frontend > ci_frontend.log
+docker-compose run --rm mcserver bash -c "source ../venv/bin/activate && coverage run --rcfile=.coveragerc tests.py && coverage combine && coverage report -m" > ci_backend.log
+./coverage_ci.sh
+cat coverage.log

docker-compose.yml

@@ -35,18 +35,6 @@ services:
     tty: true
     volumes:
       - $PWD/mc_frontend/www:/home/mc/mc_frontend/www
-  nginx:
-    command: nginx -g "daemon off;"
-    image: nginx:alpine
-    ports:
-      - "8100:80"
-    restart: always
-    volumes:
-      - $PWD/mc_frontend/www:/usr/share/nginx/html
-      - ./mc_frontend/nginx.conf:/etc/nginx/nginx.conf
-    depends_on:
-      - mc_frontend
-      - mcserver
   mcserver:
     build:
       context: ./mc_backend
@@ -63,5 +51,14 @@ services:
       - "5000:5000"
     restart: always
     stdin_open: true
+  nginx:
+    command: nginx -g "daemon off;"
+    image: nginx:alpine
+    ports:
+      - "8100:80"
+    restart: always
+    volumes:
+      - $PWD/mc_frontend/www:/usr/share/nginx/html
+      - ./mc_frontend/nginx.conf:/etc/nginx/nginx.conf
 volumes:
   db-data:

mc_backend/.coveragerc

@@ -13,5 +13,6 @@ parallel = True
 exclude_lines =
     # Don't complain if non-runnable code isn't run:
     if __name__ == .__main__.:
+    except ModuleNotFoundError:
 ignore_errors = True
 show_missing = True

mc_backend/app.py

@@ -4,4 +4,4 @@ from mcserver import get_app, get_cfg
 app: Flask = get_app()
 
 if __name__ == "__main__":
-    app.run(host=get_cfg().HOST_IP, port=get_cfg().HOST_PORT, use_reloader=False)
+    app.run(host=get_cfg().HOST_IP_MCSERVER, port=get_cfg().HOST_PORT, use_reloader=False)

mc_backend/csm/__init__.py

@@ -20,7 +20,7 @@ def get_cfg() -> Type[Config]:
 
 def run_app() -> None:
     cfg: Type[Config] = get_cfg()
-    get_app().run(host=cfg.HOST_IP, port=cfg.CORPUS_STORAGE_MANAGER_PORT, use_reloader=False)
+    get_app().run(host=cfg.HOST_IP_CSM, port=cfg.CORPUS_STORAGE_MANAGER_PORT, use_reloader=False)
 
 
 if __name__ == "__main__":

mc_backend/csm/__main__.py

 from csm import get_app, get_cfg
 
-get_app().run(host=get_cfg().HOST_IP, port=get_cfg().HOST_PORT, use_reloader=False)
+get_app().run(host=get_cfg().HOST_IP_CSM, port=get_cfg().HOST_PORT, use_reloader=False)

mc_backend/csm/app/api/annisFindAPI.py

 import flask
-from flask_restful import Resource, reqparse
-from mcserver.app.services import NetworkService, CorpusService, AnnotationService
+from flask_restful import Resource
+from flask_restful.reqparse import RequestParser
+
+from mcserver.app.services import NetworkService, CorpusService
 
 
 class AnnisFindAPI(Resource):
     def __init__(self):
-        self.reqparse = reqparse.RequestParser()
+        self.reqparse: RequestParser = NetworkService.base_request_parser.copy()
         self.reqparse.add_argument("aql", type=str, required=True, location="form", help="No AQL provided")
         self.reqparse.add_argument("urn", type=str, required=True, default="", location="form", help="No URN provided")
         super(AnnisFindAPI, self).__init__()

mc_backend/csm/app/api/corpusStorageManagerAPI.py

@@ -4,7 +4,8 @@ from typing import Dict, List
 
 import flask
 from conllu import TokenList
-from flask_restful import Resource, reqparse, abort
+from flask_restful import Resource, abort
+from flask_restful.reqparse import RequestParser
 
 from mcserver.app.models import ExerciseType, Phenomenon, AnnisResponse
 from mcserver.app.services import CorpusService, NetworkService
@@ -16,7 +17,7 @@ class CorpusStorageManagerAPI(Resource):
     It manages the database and everything corpus-related."""
 
     def __init__(self):
-        self.reqparse = reqparse.RequestParser()
+        self.reqparse: RequestParser = NetworkService.base_request_parser.copy()
         self.reqparse.add_argument("title", type=str, required=True, location="data", help="No title provided")
         self.reqparse.add_argument("annotations", required=True, location="data",
                                    help="No annotations provided")

mc_backend/csm/app/api/frequencyAPI.py

 from typing import List, Dict, Set
 import flask
-from flask_restful import Resource, reqparse
+from flask_restful import Resource
+from flask_restful.reqparse import RequestParser
+
 from mcserver.app.models import FrequencyAnalysis, Phenomenon
 from mcserver.app.services import NetworkService, CorpusService, AnnotationService
 
 
 class FrequencyAPI(Resource):
     def __init__(self):
-        self.reqparse = reqparse.RequestParser()
+        self.reqparse: RequestParser = NetworkService.base_request_parser.copy()
         self.reqparse.add_argument("urn", type=str, required=True, default="", location="form", help="No URN provided")
         super(FrequencyAPI, self).__init__()
 

mc_backend/csm/app/api/subgraphAPI.py

@@ -2,7 +2,9 @@ import json
 from typing import Dict, List
 
 import flask
-from flask_restful import Resource, reqparse
+from flask_restful import Resource
+from flask_restful.reqparse import RequestParser
+
 from mcserver.app.models import ExerciseData, GraphData, Solution, SolutionElement, AnnisResponse
 
 from mcserver.app.services import CorpusService, AnnotationService, NetworkService
@@ -10,7 +12,7 @@ from mcserver.app.services import CorpusService, AnnotationService, NetworkServi
 
 class SubgraphAPI(Resource):
     def __init__(self):
-        self.reqparse = reqparse.RequestParser()
+        self.reqparse: RequestParser = NetworkService.base_request_parser.copy()
         self.reqparse.add_argument("aqls", required=False, location="data", help="No AQLs provided", action="append")
         self.reqparse.add_argument("ctx_left", type=str, required=False, default="", location="data",
                                    help="No left context provided")

mc_backend/csm/app/api/textcomplexityAPI.py

 import rapidjson as json
 
 import flask
-from flask_restful import Resource, reqparse
+from flask_restful import Resource
+from flask_restful.reqparse import RequestParser
 
 from mcserver.app.models import AnnisResponse, GraphData, TextComplexity
 from mcserver.app.services import NetworkService, CorpusService, TextComplexityService
@@ -11,7 +12,7 @@ class TextComplexityAPI(Resource):
     """The Text Complexity API resource. It gives users measures for text complexity for a given text."""
 
     def __init__(self):
-        self.reqparse = reqparse.RequestParser()
+        self.reqparse: RequestParser = NetworkService.base_request_parser.copy()
         self.reqparse.add_argument('urn', type=str, location="data", required=True, help='No URN provided')
         self.reqparse.add_argument('measure', type=str, location="data", required=True, help='No MEASURE provided')
         self.reqparse.add_argument('annis_response', type=dict, location="data", required=False,

mc_backend/csm/gunicorn_config.py

 """Configuration for the gunicorn server"""
 from mcserver import Config
 
-bind = "{0}:{1}".format(Config.HOST_IP, Config.CORPUS_STORAGE_MANAGER_PORT)
+bind = "{0}:{1}".format(Config.HOST_IP_CSM, Config.CORPUS_STORAGE_MANAGER_PORT)
 debug = False
 reload = True
 timeout = 3600

mc_backend/mcserver/__init__.py

@@ -20,4 +20,4 @@ def get_cfg() -> Type[Config]:
 
 if __name__ == "__main__":
     # reloader has to be disabled because of a bug with Flask and multiprocessing
-    get_app().run(host=get_cfg().HOST_IP, port=get_cfg().HOST_PORT, use_reloader=False)
+    get_app().run(host=get_cfg().HOST_IP_MCSERVER, port=get_cfg().HOST_PORT, use_reloader=False)

mc_backend/mcserver/__main__.py

 from mcserver import get_app, get_cfg
 
-get_app().run(host=get_cfg().HOST_IP, port=get_cfg().HOST_PORT, use_reloader=False)
+get_app().run(host=get_cfg().HOST_IP_MCSERVER, port=get_cfg().HOST_PORT, use_reloader=False)

mc_backend/mcserver/app/__init__.py

@@ -5,7 +5,9 @@ from logging.handlers import RotatingFileHandler
 from threading import Thread
 from time import strftime
 from typing import Type
-from flask import Flask, got_request_exception, request, Response
+
+import flask
+from flask import Flask, got_request_exception, request, Response, send_from_directory
 from flask_cors import CORS
 from flask_migrate import Migrate
 from flask_sqlalchemy import SQLAlchemy
@@ -59,6 +61,12 @@ def init_app_common(cfg: Type[Config] = Config, is_csm: bool = False) -> Flask:
                         request.full_path, response.status)
         return response
 
+    @app.route(Config.SERVER_URI_FAVICON)
+    def get_favicon():
+        """Sends the favicon to browsers, which is used, e.g., in the tabs as a symbol for our application."""
+        mime_type: str = 'image/vnd.microsoft.icon'
+        return send_from_directory(Config.ASSETS_DIRECTORY, Config.FAVICON_FILE_NAME, mimetype=mime_type)
+
     @app.teardown_appcontext
     def shutdown_session(exception=None):
         """ Shuts down the session when the application exits. (maybe also after every request ???) """
@@ -100,7 +108,7 @@ def log_exception(sender_app: Flask, exception, **extra):
         **extra -- any additional arguments
     """
     # TODO: RETURN ERROR IN JSON FORMAT
-    sender_app.logger.exception("ERROR")
+    sender_app.logger.exception(f"ERROR for {flask.request.url}")
 
 
 def start_updater(app: Flask) -> Thread:

mc_backend/mcserver/app/api/corpusAPI.py

 """The corpus API. Add it to your REST API to provide users with metadata about specific texts."""
-from flask_restful import Resource, reqparse, abort, marshal
+from flask_restful import Resource, abort, marshal
+from flask_restful.reqparse import RequestParser
 
 from mcserver.app import db
 from mcserver.app.models import Corpus, corpus_fields
+from mcserver.app.services import NetworkService
 
 
 class CorpusAPI(Resource):
@@ -10,7 +12,7 @@ class CorpusAPI(Resource):
 
     def __init__(self):
         """Initialize possible arguments for calls to the corpus REST API."""
-        self.reqparse = reqparse.RequestParser()
+        self.reqparse: RequestParser = NetworkService.base_request_parser.copy()
         self.reqparse.add_argument("title", type=str, required=False, help="No title provided")
         self.reqparse.add_argument("author", type=str, required=False, help="No author provided")
         self.reqparse.add_argument("source_urn", type=str, required=False, help="No source URN provided")

mc_backend/mcserver/app/api/corpusListAPI.py

@@ -2,12 +2,13 @@
 from datetime import datetime
 
 from flask import jsonify
-from flask_restful import Resource, reqparse, marshal
+from flask_restful import Resource, marshal
+from flask_restful.reqparse import RequestParser
 from sqlalchemy.exc import OperationalError, InvalidRequestError
 
 from mcserver.app import db
 from mcserver.app.models import UpdateInfo, ResourceType, Corpus, corpus_fields
-from mcserver.app.services import CorpusService
+from mcserver.app.services import CorpusService, NetworkService
 
 
 class CorpusListAPI(Resource):
@@ -15,7 +16,7 @@ class CorpusListAPI(Resource):
 
     def __init__(self):
         """Initialize possible arguments for calls to the corpus list REST API."""
-        self.reqparse = reqparse.RequestParser()
+        self.reqparse: RequestParser = NetworkService.base_request_parser.copy()
         self.reqparse.add_argument("last_update_time", type=int, required=True,
                                    help="No milliseconds time for last update provided")
         super(CorpusListAPI, self).__init__()

mc_backend/mcserver/app/api/exerciseAPI.py

@@ -6,7 +6,8 @@ import rapidjson as json
 from typing import List, Dict
 
 import requests
-from flask_restful import Resource, reqparse, marshal, abort
+from flask_restful import Resource, marshal, abort
+from flask_restful.reqparse import RequestParser
 
 from mcserver.app import db
 from mcserver.app.models import ExerciseType, Solution, ExerciseData, Exercise, exercise_fields, AnnisResponse, \
@@ -21,7 +22,7 @@ class ExerciseAPI(Resource):
     def __init__(self):
         """Initialize possible arguments for calls to the exercise REST API."""
         # TODO: switch to other request parser, e.g. Marshmallow, because the one used by Flask-RESTful does not allow parsing arguments from different locations, e.g. one argument from 'location=args' and another argument from 'location=form'
-        self.reqparse = reqparse.RequestParser()
+        self.reqparse: RequestParser = NetworkService.base_request_parser.copy()
         self.reqparse.add_argument("urn", type=str, required=False, location="form", help="No URN provided")
         self.reqparse.add_argument("type", type=str, required=False, location="form", help="No exercise type provided")
         self.reqparse.add_argument("search_values", type=str, required=False, location="form",
@@ -49,6 +50,8 @@ class ExerciseAPI(Resource):
         ar: AnnisResponse = CorpusService.get_corpus(cts_urn=exercise.urn, is_csm=False)
         if not ar.nodes:
             abort(404)
+        exercise.last_access_time = datetime.utcnow()
+        db.session.commit()
         exercise_type: ExerciseType = ExerciseType(exercise.exercise_type)
         ar.solutions = json.loads(exercise.solutions)
         ar.uri = exercise.uri

mc_backend/mcserver/app/api/exerciseListAPI.py

@@ -4,7 +4,9 @@ from typing import List, Set
 
 import conllu
 from conllu import TokenList
-from flask_restful import Resource, reqparse
+from flask_restful import Resource
+from flask_restful.reqparse import RequestParser
+
 from mcserver.app.models import Exercise, Language, VocabularyCorpus, UpdateInfo, ResourceType
 from mcserver.app.services import NetworkService, FileService
 
@@ -14,7 +16,7 @@ class ExerciseListAPI(Resource):
 
     def __init__(self):
         """Initialize possible arguments for calls to the exercise list REST API."""
-        self.reqparse = reqparse.RequestParser()
+        self.reqparse: RequestParser = NetworkService.base_request_parser.copy()
         self.reqparse.add_argument("lang", type=str, required=True, help="No language specified")
         self.reqparse.add_argument("last_update_time", type=int, required=False, default=0,
                                    help="No milliseconds time for last update provided")

mc_backend/mcserver/app/api/fileAPI.py

@@ -7,7 +7,8 @@ from typing import List, Union
 
 import flask
 from flask import send_from_directory
-from flask_restful import Resource, reqparse, abort
+from flask_restful import Resource, abort
+from flask_restful.reqparse import RequestParser
 from werkzeug.wrappers import ETagResponseMixin
 
 from mcserver.app import db
@@ -22,7 +23,7 @@ class FileAPI(Resource):
 
     def __init__(self):
         """Initialize possible arguments for calls to the file REST API."""
-        self.reqparse = reqparse.RequestParser()
+        self.reqparse: RequestParser = NetworkService.base_request_parser.copy()
         self.reqparse.add_argument("id", type=str, required=False, location="args",
                                    help="No exercise ID or URN provided")
         self.reqparse.add_argument("type", type=str, required=False, location="args", help="No file type provided")

mc_backend/mcserver/app/api/frequencyAPI.py

 import flask
 import requests
-from flask_restful import Resource, reqparse
+from flask_restful import Resource
 import rapidjson as json
+from flask_restful.reqparse import RequestParser
 
 from mcserver import Config
 from mcserver.app.services import NetworkService
@@ -10,7 +11,7 @@ from mcserver.app.services import NetworkService
 class FrequencyAPI(Resource):
     def __init__(self):
         # TODO: FIX THE REQUEST PARSING FOR ALL APIs
-        self.reqparse = reqparse.RequestParser()
+        self.reqparse: RequestParser = NetworkService.base_request_parser.copy()
         self.reqparse.add_argument("urn", type=str, required=True, default="", location="form", help="No URN provided")
         super(FrequencyAPI, self).__init__()
 

mc_backend/mcserver/app/api/h5pAPI.py

 import json
 from typing import List
 
-from flask_restful import Resource, reqparse, abort
+from flask_restful import Resource, abort
+from flask_restful.reqparse import RequestParser
 
 from mcserver.app.models import Language, Exercise, ExerciseType, Solution
 from mcserver.app.services import TextService, NetworkService
@@ -12,7 +13,7 @@ class H5pAPI(Resource):
 
     def __init__(self):
         """Initialize possible arguments for calls to the H5P REST API."""
-        self.reqparse = reqparse.RequestParser()
+        self.reqparse: RequestParser = NetworkService.base_request_parser.copy()
         self.reqparse.add_argument("eid", type=str, required=True, default="", help="No exercise ID provided")
         self.reqparse.add_argument("lang", type=str, required=True, default="en", help="No language code provided")
         self.reqparse.add_argument("solution_indices", type=str, required=False, help="No solution IDs provided")

mc_backend/mcserver/app/api/kwicAPI.py

@@ -10,7 +10,8 @@ from typing import List, Dict
 import requests