diff --git a/docker-compose.yml b/docker-compose.yml index f42eb4ebf7ef63d298c3f029597b1da489167b10..260c2e371b44459de70ff74b93639224005d896c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,7 +13,6 @@ services: - IS_THIS_A_DOCKER_CONTAINER=Yes - PYTHONPATH=/home/mc ports: - - "8122:22" - "6555:6555" restart: always stdin_open: true @@ -21,6 +20,8 @@ services: image: postgres environment: - POSTGRES_HOST_AUTH_METHOD=trust +# ports: +# - "5432:5432" restart: always volumes: - db-data:/var/lib/postgresql/data @@ -47,7 +48,6 @@ services: - IS_THIS_A_DOCKER_CONTAINER=Yes - PYTHONPATH=/home/mc ports: - - "8022:22" - "5000:5000" restart: always stdin_open: true diff --git a/mc_backend/.coveragerc b/mc_backend/.coveragerc index 88a16b08863b062749c2cf67a3708e429fcd8e3e..badc0f3f1072895fee55358fd7ef6b819c64902a 100644 --- a/mc_backend/.coveragerc +++ b/mc_backend/.coveragerc @@ -6,6 +6,8 @@ cover_pylib = False omit = */site-packages/* */migrations/* + # cannot run tests for files that are generated and updated automatically + */models_auto.py parallel = True [report] diff --git a/mc_backend/README.md b/mc_backend/README.md index 55fd9674cf36b569edae8e3446d51f40cf7d4f3e..2572636d78072c241f7dd3c98e3d8af45368f472 100644 --- a/mc_backend/README.md +++ b/mc_backend/README.md @@ -34,13 +34,14 @@ Or combine both commands in one line: `pip list -o --format=freeze | grep -v '^\ ---------------------------------------------------------------- # Database -To autogenerate a new migration script, start the Docker container with the database and run: `flask db migrate`. - -To migrate the database to a newer version manually, run: `flask db upgrade` - -To migrate the database to a newer version manually, run: `flask db downgrade` - -If it does nothing or fails, make sure that the environment variable FLASK_APP is set correctly (see https://flask.palletsprojects.com/en/1.1.x/cli/). +To autogenerate a new migration script: +1. Start the Docker container with the database: `docker-compose run -p 5432:5432 -d db` +2. Create a new migration: `flask db migrate`. +3. Perform a migration... + - ... to a newer version: `flask db upgrade`. + - ... to an older version: `flask db downgrade`. + - If it does nothing or fails, make sure that the environment variable FLASK_APP is set correctly (see https://flask.palletsprojects.com/en/1.1.x/cli/): `export FLASK_APP=app.py` +5. To finish the process, shut down the database container: `docker-compose down` ---------------------------------------------------------------- diff --git a/mc_backend/app.py b/mc_backend/app.py index ddd570f3409b4d5f3b713da267ea06535aef3e74..3fb9c8fe5a38216a8063ccdaaf76ef4e34acb896 100644 --- a/mc_backend/app.py +++ b/mc_backend/app.py @@ -1,4 +1,5 @@ from flask import Flask + from mcserver import get_app, get_cfg app: Flask = get_app() diff --git a/mc_backend/csm/app/__init__.py b/mc_backend/csm/app/__init__.py index 9def12d58074dba83241e2bfa4fe2292af82ec40..36f8cbadef4f6c5e76389dc96fa82ce0615da301 100644 --- a/mc_backend/csm/app/__init__.py +++ b/mc_backend/csm/app/__init__.py @@ -7,7 +7,7 @@ from mcserver import Config from mcserver.app import init_app_common, init_logging -def create_csm_app(cfg: Type[Config] = Config): +def create_csm_app(cfg: Type[Config] = Config) -> Flask: """Creates a new Flask app that represents a Corpus Storage Manager.""" Config.CORPUS_STORAGE_MANAGER = CorpusStorageManager(Config.GRAPH_DATABASE_DIR) diff --git a/mc_backend/mcserver/__init__.py b/mc_backend/mcserver/__init__.py index 25828bb6ebbbd3bd0c41f2164392b9cd5524a898..de920a63ae8b5ccb86f3bb82ba13d88d9036e135 100644 --- a/mc_backend/mcserver/__init__.py +++ b/mc_backend/mcserver/__init__.py @@ -4,9 +4,10 @@ It is a server-side backend for retrieving Latin texts and generating language exercises for them.""" import sys from typing import Type + from flask import Flask -from mcserver.app import create_app from mcserver.config import Config, ProductionConfig, TestingConfig, DevelopmentConfig +from mcserver.app import create_app def get_app() -> Flask: diff --git a/mc_backend/mcserver/app/__init__.py b/mc_backend/mcserver/app/__init__.py index 56c38cf8e5a8786f7951e66bb906bc68afa76de0..11c564a9c41b5f04027420d444b1c08db47dce9f 100644 --- a/mc_backend/mcserver/app/__init__.py +++ b/mc_backend/mcserver/app/__init__.py @@ -1,21 +1,47 @@ """The main module for the application. It contains the application factory and provides access to the database.""" import logging +import os import sys from logging.handlers import RotatingFileHandler from threading import Thread from time import strftime from typing import Type - +import connexion import flask +from connexion import FlaskApp from flask import Flask, got_request_exception, request, Response, send_from_directory from flask_cors import CORS from flask_migrate import Migrate from flask_sqlalchemy import SQLAlchemy +from open_alchemy import init_yaml from mcserver.config import Config db: SQLAlchemy = SQLAlchemy() # session_options={"autocommit": True} migrate: Migrate = Migrate(directory=Config.MIGRATIONS_DIRECTORY) +# do this _BEFORE_ you add any APIs to your application +init_yaml(Config.API_SPEC_FILE_PATH, base=db.Model, + models_filename=os.path.join(Config.MC_SERVER_DIRECTORY, "models_auto.py")) + + +def apply_event_handlers(app: FlaskApp): + """Applies event handlers to a given Flask application, such as logging after requests or teardown logic.""" + + @app.app.after_request + def after_request(response: Response) -> Response: + """ Logs metadata for every request. """ + timestamp = strftime('[%Y-%m-%d %H:%M:%S]') + app.app.logger.info('%s %s %s %s %s %s', timestamp, request.remote_addr, request.method, request.scheme, + request.full_path, response.status) + return response + + @app.route(Config.SERVER_URI_FAVICON) + def get_favicon(): + """Sends the favicon to browsers, which is used, e.g., in the tabs as a symbol for our application.""" + mime_type: str = 'image/vnd.microsoft.icon' + return send_from_directory(Config.ASSETS_DIRECTORY, Config.FAVICON_FILE_NAME, mimetype=mime_type) + + app.app.teardown_appcontext(shutdown_session) def create_app(cfg: Type[Config] = Config) -> Flask: @@ -26,7 +52,7 @@ def create_app(cfg: Type[Config] = Config) -> Flask: # use local postgres database for migrations if len(sys.argv) > 2 and sys.argv[2] == Config.FLASK_MIGRATE: cfg.SQLALCHEMY_DATABASE_URI = Config.DATABASE_URL_LOCAL - app = init_app_common(cfg=cfg) + app: Flask = init_app_common(cfg=cfg) from mcserver.app.services import bp as services_bp app.register_blueprint(services_bp) from mcserver.app.api import bp as api_bp @@ -51,27 +77,12 @@ def full_init(app: Flask, is_csm: bool) -> None: def init_app_common(cfg: Type[Config] = Config, is_csm: bool = False) -> Flask: """ Initializes common Flask parts, e.g. CORS, configuration, database, migrations and custom corpora.""" - app = Flask(__name__) - - @app.after_request - def after_request(response: Response) -> Response: - """ Logs metadata for every request. """ - timestamp = strftime('[%Y-%m-%d %H:%M:%S]') - app.logger.info('%s %s %s %s %s %s', timestamp, request.remote_addr, request.method, request.scheme, - request.full_path, response.status) - return response - - @app.route(Config.SERVER_URI_FAVICON) - def get_favicon(): - """Sends the favicon to browsers, which is used, e.g., in the tabs as a symbol for our application.""" - mime_type: str = 'image/vnd.microsoft.icon' - return send_from_directory(Config.ASSETS_DIRECTORY, Config.FAVICON_FILE_NAME, mimetype=mime_type) - - @app.teardown_appcontext - def shutdown_session(exception=None): - """ Shuts down the session when the application exits. (maybe also after every request ???) """ - db.session.remove() - + connexion_app: FlaskApp = connexion.FlaskApp( + __name__, port=(cfg.CORPUS_STORAGE_MANAGER_PORT if is_csm else cfg.HOST_PORT), + specification_dir=Config.MC_SERVER_DIRECTORY) + connexion_app.add_api(Config.API_SPEC_FILE_PATH, arguments={'title': 'Machina Callida Backend REST API'}) + apply_event_handlers(connexion_app) + app: Flask = connexion_app.app # allow CORS requests for all API routes CORS(app) # , resources=r"/*" app.config.from_object(cfg) @@ -120,6 +131,12 @@ def start_updater(app: Flask) -> Thread: return t +def shutdown_session(exception=None): + """ Shuts down the session when the application exits. (maybe also after every request ???) """ + db.session.remove() + + # import the models so we can access them from other parts of the app using imports from "app.models"; # this has to be at the bottom of the file from mcserver.app import models +from mcserver.app import api diff --git a/mc_backend/mcserver/app/api/__init__.py b/mc_backend/mcserver/app/api/__init__.py index 7af7f82ed12425bc30e8fa9501afcfb3fbccb032..d8b7a16727f623b0f87e1061714704de0549d5fc 100644 --- a/mc_backend/mcserver/app/api/__init__.py +++ b/mc_backend/mcserver/app/api/__init__.py @@ -1,15 +1,12 @@ """The API blueprint. Register it on the main application to enable the REST API for text retrieval.""" from flask import Blueprint from flask_restful import Api - from mcserver import Config bp = Blueprint("api", __name__) api = Api(bp) -from mcserver.app.api.corpusAPI import CorpusAPI -from mcserver.app.api.corpusListAPI import CorpusListAPI -from mcserver.app.api.exerciseAPI import ExerciseAPI +from . import corpusAPI, corpusListAPI, exerciseAPI from mcserver.app.api.exerciseListAPI import ExerciseListAPI from mcserver.app.api.fileAPI import FileAPI from mcserver.app.api.frequencyAPI import FrequencyAPI @@ -22,9 +19,6 @@ from mcserver.app.api.validReffAPI import ValidReffAPI from mcserver.app.api.vectorNetworkAPI import VectorNetworkAPI from mcserver.app.api.vocabularyAPI import VocabularyAPI -api.add_resource(CorpusListAPI, Config.SERVER_URI_CORPORA, endpoint="corpora") -api.add_resource(CorpusAPI, Config.SERVER_URI_CORPORA + "/", endpoint="corpus") -api.add_resource(ExerciseAPI, Config.SERVER_URI_EXERCISE, endpoint="exercise") api.add_resource(ExerciseListAPI, Config.SERVER_URI_EXERCISE_LIST, endpoint="exerciseList") api.add_resource(FileAPI, Config.SERVER_URI_FILE, endpoint="file") api.add_resource(FrequencyAPI, Config.SERVER_URI_FREQUENCY, endpoint="frequency") diff --git a/mc_backend/mcserver/app/api/corpusAPI.py b/mc_backend/mcserver/app/api/corpusAPI.py index 31f4dd488cf1c7e429186e2b483c1c3cf5084b82..aa2cdff7ea496a3cf8f1d1b2daa0aefbe314f4df 100644 --- a/mc_backend/mcserver/app/api/corpusAPI.py +++ b/mc_backend/mcserver/app/api/corpusAPI.py @@ -1,47 +1,41 @@ """The corpus API. Add it to your REST API to provide users with metadata about specific texts.""" -from flask_restful import Resource, abort, marshal -from flask_restful.reqparse import RequestParser +from typing import Union +import connexion +from connexion.lifecycle import ConnexionResponse +from flask import Response + +from mcserver import Config from mcserver.app import db -from mcserver.app.models import Corpus, corpus_fields from mcserver.app.services import NetworkService +from mcserver.models_auto import Corpus + + +def delete(cid: int) -> Union[Response, ConnexionResponse]: + """The DELETE method for the corpus REST API. It deletes metadata for a specific text.""" + corpus: Corpus = db.session.query(Corpus).filter_by(cid=cid).first() + if corpus is None: + return connexion.problem(404, Config.ERROR_TITLE_NOT_FOUND, Config.ERROR_MESSAGE_CORPUS_NOT_FOUND) + db.session.delete(corpus) + db.session.commit() + return NetworkService.make_json_response(True) + + +def get(cid: int) -> Union[Response, ConnexionResponse]: + """The GET method for the corpus REST API. It provides metadata for a specific text.""" + corpus: Corpus = db.session.query(Corpus).filter_by(cid=cid).first() + if corpus is None: + return connexion.problem(404, Config.ERROR_TITLE_NOT_FOUND, Config.ERROR_MESSAGE_CORPUS_NOT_FOUND) + return NetworkService.make_json_response(corpus.to_dict()) -class CorpusAPI(Resource): - """The corpus API resource. It enables some of the CRUD operations for metadata about specific texts.""" - - def __init__(self): - """Initialize possible arguments for calls to the corpus REST API.""" - self.reqparse: RequestParser = NetworkService.base_request_parser.copy() - self.reqparse.add_argument("title", type=str, required=False, help="No title provided") - self.reqparse.add_argument("author", type=str, required=False, help="No author provided") - self.reqparse.add_argument("source_urn", type=str, required=False, help="No source URN provided") - super(CorpusAPI, self).__init__() - - def get(self, cid): - """The GET method for the corpus REST API. It provides metadata for a specific text.""" - corpus: Corpus = Corpus.query.filter_by(cid=cid).first() - if corpus is None: - abort(404) - return {"corpus": marshal(corpus, corpus_fields)} - - def put(self, cid): - """The PUT method for the corpus REST API. It provides updates metadata for a specific text.""" - corpus: Corpus = Corpus.query.filter_by(cid=cid).first() - if corpus is None: - abort(404) - args = self.reqparse.parse_args() - for k, v in args.items(): - if v is not None: - setattr(corpus, k, v) - db.session.commit() - return {"corpus": marshal(corpus, corpus_fields)} - - def delete(self, cid): - """The DELETE method for the corpus REST API. It deletes metadata for a specific text.""" - corpus: Corpus = Corpus.query.filter_by(cid=cid).first() - if corpus is None: - abort(404) - db.session.delete(corpus) - db.session.commit() - return {"result": True} +def patch(cid: int, **kwargs) -> Union[Response, ConnexionResponse]: + """The PUT method for the corpus REST API. It provides updates metadata for a specific text.""" + corpus: Corpus = db.session.query(Corpus).filter_by(cid=cid).first() + if corpus is None: + return connexion.problem(404, Config.ERROR_TITLE_NOT_FOUND, Config.ERROR_MESSAGE_CORPUS_NOT_FOUND) + for k, v in kwargs.items(): + if v is not None: + setattr(corpus, k, v) + db.session.commit() + return NetworkService.make_json_response(corpus.to_dict()) diff --git a/mc_backend/mcserver/app/api/corpusListAPI.py b/mc_backend/mcserver/app/api/corpusListAPI.py index fc6cb513c1ce1e9d9097b711670e9ee99e061eb3..c765bff78a8a47442ccdb5e89721ccb63a24942d 100644 --- a/mc_backend/mcserver/app/api/corpusListAPI.py +++ b/mc_backend/mcserver/app/api/corpusListAPI.py @@ -1,38 +1,25 @@ """The corpus list API. Add it to your REST API to provide users with a list of metadata for available texts.""" from datetime import datetime -from flask import jsonify -from flask_restful import Resource, marshal -from flask_restful.reqparse import RequestParser +from connexion.lifecycle import ConnexionResponse +from flask import Response from sqlalchemy.exc import OperationalError, InvalidRequestError - +from typing import List, Union from mcserver.app import db -from mcserver.app.models import UpdateInfo, ResourceType, Corpus, corpus_fields -from mcserver.app.services import CorpusService, NetworkService - - -class CorpusListAPI(Resource): - """The corpus list API resource. It enables some of the CRUD operations for a list of metadata about all texts.""" +from mcserver.app.models import ResourceType +from mcserver.app.services import NetworkService +from mcserver.models_auto import Corpus, UpdateInfo - def __init__(self): - """Initialize possible arguments for calls to the corpus list REST API.""" - self.reqparse: RequestParser = NetworkService.base_request_parser.copy() - self.reqparse.add_argument("last_update_time", type=int, required=True, - help="No milliseconds time for last update provided") - super(CorpusListAPI, self).__init__() - def get(self): - """The GET method for the corpus list REST API. It provides metadata for all available texts.""" - args = self.reqparse.parse_args() - last_update: int = args["last_update_time"] - last_update_time: datetime = datetime.fromtimestamp(last_update / 1000.0) - ui_cts: UpdateInfo - try: - ui_cts = UpdateInfo.query.filter_by(resource_type=ResourceType.cts_data.name).first() - except (InvalidRequestError, OperationalError): - db.session.rollback() - return None - if ui_cts.last_modified_time >= last_update_time: - CorpusService.existing_corpora = Corpus.query.all() - return jsonify({"corpora": [marshal(corpus, corpus_fields) for corpus in CorpusService.existing_corpora]}) - return None +def get(last_update_time: int) -> Union[Response, ConnexionResponse]: + """The GET method for the corpus list REST API. It provides metadata for all available texts.""" + ui_cts: UpdateInfo + try: + ui_cts = db.session.query(UpdateInfo).filter_by(resource_type=ResourceType.cts_data.name).first() + except (InvalidRequestError, OperationalError): + db.session.rollback() + return NetworkService.make_json_response(None) + if ui_cts.last_modified_time >= last_update_time / 1000: + corpora: List[Corpus] = db.session.query(Corpus).all() + return NetworkService.make_json_response([x.to_dict() for x in corpora]) + return NetworkService.make_json_response(None) diff --git a/mc_backend/mcserver/app/api/exerciseAPI.py b/mc_backend/mcserver/app/api/exerciseAPI.py index 7b9249fd074768546b5c74d6a6e0b461a80f7979..1ac4bcd9148dc3097c26fe48e72b9679d437859b 100644 --- a/mc_backend/mcserver/app/api/exerciseAPI.py +++ b/mc_backend/mcserver/app/api/exerciseAPI.py @@ -1,85 +1,17 @@ import uuid -from collections import OrderedDict from datetime import datetime - +import connexion import rapidjson as json -from typing import List, Dict - +from typing import List, Dict, Union import requests -from flask_restful import Resource, marshal, abort -from flask_restful.reqparse import RequestParser - +from connexion.lifecycle import ConnexionResponse +from flask import Response from mcserver.app import db -from mcserver.app.models import ExerciseType, Solution, ExerciseData, Exercise, exercise_fields, AnnisResponse, \ - Phenomenon, TextComplexity, TextComplexityMeasure, UpdateInfo, ResourceType +from mcserver.app.models import ExerciseType, Solution, ExerciseData, AnnisResponse, Phenomenon, TextComplexity, \ + TextComplexityMeasure, ResourceType, ExerciseMC from mcserver.app.services import AnnotationService, CorpusService, NetworkService, TextComplexityService from mcserver.config import Config - - -class ExerciseAPI(Resource): - """The exercise API resource. It creates exercises for a given text.""" - - def __init__(self): - """Initialize possible arguments for calls to the exercise REST API.""" - # TODO: switch to other request parser, e.g. Marshmallow, because the one used by Flask-RESTful does not allow parsing arguments from different locations, e.g. one argument from 'location=args' and another argument from 'location=form' - self.reqparse: RequestParser = NetworkService.base_request_parser.copy() - self.reqparse.add_argument("urn", type=str, required=False, location="form", help="No URN provided") - self.reqparse.add_argument("type", type=str, required=False, location="form", help="No exercise type provided") - self.reqparse.add_argument("search_values", type=str, required=False, location="form", - help="No search value provided") - self.reqparse.add_argument("type_translation", type=str, location="form", required=False, - help="No exercise type translation provided") - self.reqparse.add_argument("work_author", type=str, location="form", required=False, - help="No work_author provided", default="") - self.reqparse.add_argument("work_title", type=str, required=False, location="form", - help="No work title provided", default="") - self.reqparse.add_argument("instructions", type=str, required=False, location="form", default="") - self.reqparse.add_argument("general_feedback", type=str, required=False, location="form", default=" ") - self.reqparse.add_argument("correct_feedback", type=str, required=False, location="form", default=" ") - self.reqparse.add_argument("partially_correct_feedback", type=str, required=False, location="form", default=" ") - self.reqparse.add_argument("incorrect_feedback", type=str, required=False, location="form", default=" ") - self.reqparse.add_argument("eid", type=str, required=False, location="args", help="No exercise ID provided") - super(ExerciseAPI, self).__init__() - - def get(self): - args: dict = self.reqparse.parse_args() - eid: str = args["eid"] - exercise: Exercise = Exercise.query.filter_by(eid=eid).first() - if exercise is None: - abort(404) - ar: AnnisResponse = CorpusService.get_corpus(cts_urn=exercise.urn, is_csm=False) - if not ar.nodes: - abort(404) - exercise.last_access_time = datetime.utcnow() - db.session.commit() - exercise_type: ExerciseType = ExerciseType(exercise.exercise_type) - ar.solutions = json.loads(exercise.solutions) - ar.uri = exercise.uri - ar.exercise_id = exercise.eid - ar.exercise_type = exercise_type.value - return NetworkService.make_json_response(ar.__dict__) - - def post(self): - # get request arguments - args: dict = self.reqparse.parse_args() - urn: str = args["urn"] - exercise_type: ExerciseType = ExerciseType(args["type"]) - search_values_json: str = args["search_values"] - search_values_list: List[str] = json.loads(search_values_json) - aqls: List[str] = AnnotationService.map_search_values_to_aql(search_values_list=search_values_list, - exercise_type=exercise_type) - search_phenomena: List[Phenomenon] = [Phenomenon[x.split("=")[0]] for x in search_values_list] - # if there is custom text instead of a URN, immediately annotate it - conll_string_or_urn: str = urn if CorpusService.is_urn(urn) else AnnotationService.get_udpipe( - CorpusService.get_raw_text(urn, False)) - # construct graph from CONLL data - response: dict = get_graph_data(title=urn, conll_string_or_urn=conll_string_or_urn, aqls=aqls, - exercise_type=exercise_type, search_phenomena=search_phenomena) - solutions_dict_list: List[Dict] = response["solutions"] - solutions: List[Solution] = [Solution(json_dict=x) for x in solutions_dict_list] - ar: AnnisResponse = make_new_exercise(graph_data_raw=response["graph_data_raw"], solutions=solutions, args=args, - conll=response["conll"], search_values=args["search_values"], urn=urn) - return NetworkService.make_json_response(ar.__dict__) +from mcserver.models_auto import Exercise, TExercise, UpdateInfo def adjust_solutions(exercise_data: ExerciseData, exercise_type: str, solutions: List[Solution]) -> List[Solution]: @@ -93,6 +25,23 @@ def adjust_solutions(exercise_data: ExerciseData, exercise_type: str, solutions: return solutions +def get(eid: str) -> Union[Response, ConnexionResponse]: + exercise: TExercise = db.session.query(Exercise).filter_by(eid=eid).first() + if exercise is None: + return connexion.problem(404, Config.ERROR_TITLE_NOT_FOUND, Config.ERROR_MESSAGE_EXERCISE_NOT_FOUND) + ar: AnnisResponse = CorpusService.get_corpus(cts_urn=exercise.urn, is_csm=False) + if not ar.nodes: + return connexion.problem(404, Config.ERROR_TITLE_NOT_FOUND, Config.ERROR_MESSAGE_CORPUS_NOT_FOUND) + exercise.last_access_time = datetime.utcnow().timestamp() + db.session.commit() + exercise_type: ExerciseType = ExerciseType(exercise.exercise_type) + ar.solutions = json.loads(exercise.solutions) + ar.uri = NetworkService.get_exercise_uri(exercise) + ar.exercise_id = exercise.eid + ar.exercise_type = exercise_type.value + return NetworkService.make_json_response(ar.__dict__) + + def get_graph_data(title: str, conll_string_or_urn: str, aqls: List[str], exercise_type: ExerciseType, search_phenomena: List[Phenomenon]): """Sends annotated text data or a URN to the Corpus Storage Manager in order to get a graph.""" @@ -104,38 +53,35 @@ def get_graph_data(title: str, conll_string_or_urn: str, aqls: List[str], exerci try: return json.loads(response.text) except ValueError: - abort(500) + raise -def make_new_exercise(solutions: List[Solution], args: dict, search_values: str, graph_data_raw: dict, - conll: str, urn: str) -> AnnisResponse: +def make_new_exercise(conll: str, correct_feedback: str, exercise_type: str, general_feedback: str, + graph_data_raw: dict, incorrect_feedback: str, instructions: str, partially_correct_feedback: str, + search_values: str, solutions: List[Solution], type_translation: str, urn: str, + work_author: str, work_title: str) -> AnnisResponse: """ Creates a new exercise and makes it JSON serializable. """ # generate a GUID so we can offer the exercise XML as a file download xml_guid = str(uuid.uuid4()) # assemble the mapped exercise data ed: ExerciseData = AnnotationService.map_graph_data_to_exercise(graph_data_raw=graph_data_raw, solutions=solutions, xml_guid=xml_guid) - exercise_type = args["type"] # for markWords exercises, add the maximum number of correct solutions to the description - instructions: str = args["instructions"] + ( - f"({len(solutions)})" if exercise_type == ExerciseType.markWords.value else "") + instructions += (f"({len(solutions)})" if exercise_type == ExerciseType.markWords.value else "") # map the exercise data to our database data model new_exercise: Exercise = map_exercise_data_to_database(solutions=solutions, exercise_data=ed, exercise_type=exercise_type, instructions=instructions, - xml_guid=xml_guid, correct_feedback=args["correct_feedback"], - partially_correct_feedback=args[ - "partially_correct_feedback"], - incorrect_feedback=args["incorrect_feedback"], - general_feedback=args["general_feedback"], - exercise_type_translation=args.get("type_translation", ""), - conll=conll, work_author=args["work_author"], - work_title=args["work_title"], search_values=search_values, - urn=urn) - # marshal the whole object so we can get the right URI for download purposes - new_exercise_marshal: OrderedDict = marshal(new_exercise, exercise_fields) + xml_guid=xml_guid, correct_feedback=correct_feedback, + partially_correct_feedback=partially_correct_feedback, + incorrect_feedback=incorrect_feedback, + general_feedback=general_feedback, + exercise_type_translation=type_translation, conll=conll, + work_author=work_author, work_title=work_title, + search_values=search_values, urn=urn) # create a response - return AnnisResponse(solutions=json.loads(new_exercise.solutions), uri=new_exercise_marshal["uri"], - exercise_id=xml_guid) + return AnnisResponse( + solutions=json.loads(new_exercise.solutions), uri=f"{Config.SERVER_URI_FILE}/{new_exercise.eid}", + exercise_id=xml_guid) def map_exercise_data_to_database(exercise_data: ExerciseData, exercise_type: str, instructions: str, xml_guid: str, @@ -146,19 +92,53 @@ def map_exercise_data_to_database(exercise_data: ExerciseData, exercise_type: st # sort the nodes according to the ordering links AnnotationService.sort_nodes(graph_data=exercise_data.graph) # add content to solutions - solutions = adjust_solutions(exercise_data=exercise_data, solutions=solutions, exercise_type=exercise_type) + solutions: List[Solution] = adjust_solutions(exercise_data=exercise_data, solutions=solutions, + exercise_type=exercise_type) quiz_solutions: str = json.dumps([x.serialize() for x in solutions]) tc: TextComplexity = TextComplexityService.text_complexity(TextComplexityMeasure.all.name, urn, False, exercise_data.graph) - new_exercise: Exercise = Exercise(conll=conll, correct_feedback=correct_feedback, eid=xml_guid, - exercise_type=exercise_type, exercise_type_translation=exercise_type_translation, - general_feedback=general_feedback, incorrect_feedback=incorrect_feedback, - instructions=instructions, partially_correct_feedback=partially_correct_feedback, - search_values=search_values, solutions=quiz_solutions, text_complexity=tc.all, - work_author=work_author, work_title=work_title, uri=exercise_data.uri, urn=urn) + new_exercise: Exercise = ExerciseMC.from_dict( + conll=conll, correct_feedback=correct_feedback, eid=xml_guid, exercise_type=exercise_type, + exercise_type_translation=exercise_type_translation, general_feedback=general_feedback, + incorrect_feedback=incorrect_feedback, instructions=instructions, + last_access_time=datetime.utcnow().timestamp(), partially_correct_feedback=partially_correct_feedback, + search_values=search_values, solutions=quiz_solutions, text_complexity=tc.all, work_author=work_author, + work_title=work_title, urn=urn) # add the mapped exercise to the database db.session.add(new_exercise) - ui_exercises: UpdateInfo = UpdateInfo.query.filter_by(resource_type=ResourceType.exercise_list.name).first() - ui_exercises.last_modified_time = datetime.utcnow() + ui_exercises: UpdateInfo = db.session.query(UpdateInfo).filter_by( + resource_type=ResourceType.exercise_list.name).first() + ui_exercises.last_modified_time = datetime.utcnow().timestamp() db.session.commit() return new_exercise + + +def post(exercise_data: dict) -> Union[Response, ConnexionResponse]: + exercise_type: ExerciseType = ExerciseType(exercise_data["type"]) + search_values_list: List[str] = json.loads(exercise_data["search_values"]) + aqls: List[str] = AnnotationService.map_search_values_to_aql(search_values_list=search_values_list, + exercise_type=exercise_type) + search_phenomena: List[Phenomenon] = [Phenomenon[x.split("=")[0]] for x in search_values_list] + urn: str = exercise_data.get("urn", "") + # if there is custom text instead of a URN, immediately annotate it + conll_string_or_urn: str = urn if CorpusService.is_urn(urn) else AnnotationService.get_udpipe( + CorpusService.get_raw_text(urn, False)) + try: + # construct graph from CONLL data + response: dict = get_graph_data(title=urn, conll_string_or_urn=conll_string_or_urn, aqls=aqls, + exercise_type=exercise_type, search_phenomena=search_phenomena) + except ValueError: + return connexion.problem(500, Config.ERROR_TITLE_INTERNAL_SERVER_ERROR, + Config.ERROR_MESSAGE_INTERNAL_SERVER_ERROR) + solutions_dict_list: List[Dict] = response["solutions"] + solutions: List[Solution] = [Solution(json_dict=x) for x in solutions_dict_list] + ar: AnnisResponse = make_new_exercise( + conll=response["conll"], correct_feedback=exercise_data.get("correct_feedback", ""), + exercise_type=exercise_data["type"], general_feedback=exercise_data.get("general_feedback", ""), + graph_data_raw=response["graph_data_raw"], incorrect_feedback=exercise_data.get("incorrect_feedback", ""), + instructions=exercise_data["instructions"], + partially_correct_feedback=exercise_data.get("partially_correct_feedback", ""), + search_values=exercise_data["search_values"], solutions=solutions, + type_translation=exercise_data.get("type_translation", ""), urn=urn, + work_author=exercise_data.get("work_author", ""), work_title=exercise_data.get("work_title", "")) + return NetworkService.make_json_response(ar.__dict__) diff --git a/mc_backend/mcserver/app/api/exerciseListAPI.py b/mc_backend/mcserver/app/api/exerciseListAPI.py index 8ff5c4df18828e9bf88cbc5b15a66c18f1ed1eaf..0270c6533c7fd9684b06ec11ce34c2fc3caee411 100644 --- a/mc_backend/mcserver/app/api/exerciseListAPI.py +++ b/mc_backend/mcserver/app/api/exerciseListAPI.py @@ -7,8 +7,10 @@ from conllu import TokenList from flask_restful import Resource from flask_restful.reqparse import RequestParser -from mcserver.app.models import Exercise, Language, VocabularyCorpus, UpdateInfo, ResourceType +from mcserver.app import db +from mcserver.app.models import Language, VocabularyCorpus, ResourceType from mcserver.app.services import NetworkService, FileService +from mcserver.models_auto import Exercise, UpdateInfo class ExerciseListAPI(Resource): @@ -30,9 +32,9 @@ class ExerciseListAPI(Resource): args: dict = self.reqparse.parse_args() vocabulary_set: Set[str] last_update: int = args["last_update_time"] - last_update_time: datetime = datetime.fromtimestamp(last_update / 1000.0) - ui_exercises: UpdateInfo = UpdateInfo.query.filter_by(resource_type=ResourceType.exercise_list.name).first() - if ui_exercises.last_modified_time < last_update_time: + ui_exercises: UpdateInfo = db.session.query(UpdateInfo).filter_by( + resource_type=ResourceType.exercise_list.name).first() + if ui_exercises.last_modified_time < last_update / 1000: return NetworkService.make_json_response([]) try: vc: VocabularyCorpus = VocabularyCorpus[args["vocabulary"]] @@ -44,8 +46,8 @@ class ExerciseListAPI(Resource): lang = Language(args["lang"]) except ValueError: lang = Language.English - exercises: List[Exercise] = Exercise.query.filter_by(language=lang.value) - ret_val: List[dict] = [x.serialize(compress=True) for x in exercises] + exercises: List[Exercise] = db.session.query(Exercise).filter_by(language=lang.value) + ret_val: List[dict] = [NetworkService.serialize_exercise(x, compress=True) for x in exercises] matching_degrees: List[float] = [] if len(vocabulary_set): for exercise in exercises: diff --git a/mc_backend/mcserver/app/api/fileAPI.py b/mc_backend/mcserver/app/api/fileAPI.py index d12af5a40c3bbf28194fa639184c396f854647b5..ae2c604de2eef37f48db2a2c70e4b62bbb0546f8 100644 --- a/mc_backend/mcserver/app/api/fileAPI.py +++ b/mc_backend/mcserver/app/api/fileAPI.py @@ -12,10 +12,10 @@ from flask_restful.reqparse import RequestParser from werkzeug.wrappers import ETagResponseMixin from mcserver.app import db -from mcserver.app.models import Exercise, FileType, UpdateInfo, ResourceType, DownloadableFile, MimeType, \ - LearningResult, XapiStatement +from mcserver.app.models import FileType, ResourceType, DownloadableFile, MimeType, XapiStatement, LearningResultMC from mcserver.app.services import FileService, NetworkService from mcserver.config import Config +from mcserver.models_auto import Exercise, UpdateInfo, LearningResult class FileAPI(Resource): @@ -43,7 +43,7 @@ class FileAPI(Resource): clean_tmp_folder() args = self.reqparse.parse_args() eid: str = args["id"] - exercise: Exercise = Exercise.query.filter_by(eid=eid).first() + exercise: Exercise = db.session.query(Exercise).filter_by(eid=eid).first() file_type: FileType = FileType[args["type"]] file_name: str = eid + "." + file_type.value mime_type: str = MimeType[file_type.value].value @@ -52,7 +52,7 @@ class FileAPI(Resource): if not os.path.exists(os.path.join(Config.TMP_DIRECTORY, file_name)): abort(404) return send_from_directory(Config.TMP_DIRECTORY, file_name, mimetype=mime_type, as_attachment=True) - exercise.last_access_time = datetime.utcnow() + exercise.last_access_time = datetime.utcnow().timestamp() db.session.commit() solution_indices: List[int] = json.loads(args["solution_indices"] if args["solution_indices"] else "null") if solution_indices is not None: @@ -84,8 +84,9 @@ class FileAPI(Resource): def clean_tmp_folder(): """ Cleans the files directory regularly. """ - ui_file: UpdateInfo = UpdateInfo.query.filter_by(resource_type=ResourceType.file_api_clean.name).first() - if (datetime.utcnow() - ui_file.last_modified_time).total_seconds() > Config.INTERVAL_FILE_DELETE: + ui_file: UpdateInfo = db.session.query(UpdateInfo).filter_by(resource_type=ResourceType.file_api_clean.name).first() + ui_datetime: datetime = datetime.fromtimestamp(ui_file.last_modified_time) + if (datetime.utcnow() - ui_datetime).total_seconds() > Config.INTERVAL_FILE_DELETE: for file in [x for x in os.listdir(Config.TMP_DIRECTORY) if x not in ".gitignore"]: file_to_delete_type: str = os.path.splitext(file)[1].replace(".", "") file_to_delete: DownloadableFile = next((x for x in FileService.downloadable_files if @@ -94,27 +95,27 @@ def clean_tmp_folder(): if file_to_delete is not None: FileService.downloadable_files.remove(file_to_delete) os.remove(os.path.join(Config.TMP_DIRECTORY, file)) - ui_file.last_modified_time = datetime.utcnow() + ui_file.last_modified_time = datetime.utcnow().timestamp() db.session.commit() def save_learning_result(xapi_statement: XapiStatement) -> LearningResult: """Creates a new Learning Result from a XAPI Statement and saves it to the database.""" - learning_result: LearningResult = LearningResult( + learning_result: LearningResult = LearningResultMC.from_dict( actor_account_name=xapi_statement.actor.account.name, - actor_object_type=xapi_statement.actor.object_type, + actor_object_type=xapi_statement.actor.object_type.value, category_id=xapi_statement.context.context_activities.category[0].id, - category_object_type=xapi_statement.context.context_activities.category[0].object_type, - choices=xapi_statement.object.definition.choices, + category_object_type=xapi_statement.context.context_activities.category[0].object_type.value, + choices=json.dumps([x.serialize() for x in xapi_statement.object.definition.choices]), completion=xapi_statement.result.completion, - correct_responses_pattern=xapi_statement.object.definition.correct_responses_pattern, - created_time=datetime.utcnow(), + correct_responses_pattern=json.dumps(xapi_statement.object.definition.correct_responses_pattern), + created_time=datetime.utcnow().timestamp(), duration=xapi_statement.result.duration, - extensions=xapi_statement.object.definition.extensions, + extensions=json.dumps(xapi_statement.object.definition.extensions), interaction_type=xapi_statement.object.definition.interaction_type, object_definition_description=xapi_statement.object.definition.description.en_us, object_definition_type=xapi_statement.object.definition.type, - object_object_type=xapi_statement.object.object_type, + object_object_type=xapi_statement.object.object_type.value, response=xapi_statement.result.response, score_max=xapi_statement.result.score.max, score_min=xapi_statement.result.score.min, diff --git a/mc_backend/mcserver/app/api/h5pAPI.py b/mc_backend/mcserver/app/api/h5pAPI.py index efc12c94455d783bf4ab8c8ed49813e5afa5703f..cca14b68483b6a536cf8237c0e33df857715b571 100644 --- a/mc_backend/mcserver/app/api/h5pAPI.py +++ b/mc_backend/mcserver/app/api/h5pAPI.py @@ -4,8 +4,10 @@ from typing import List from flask_restful import Resource, abort from flask_restful.reqparse import RequestParser -from mcserver.app.models import Language, Exercise, ExerciseType, Solution +from mcserver.app import db +from mcserver.app.models import Language, ExerciseType, Solution from mcserver.app.services import TextService, NetworkService +from mcserver.models_auto import Exercise class H5pAPI(Resource): @@ -70,7 +72,7 @@ class H5pAPI(Resource): lang = Language(args["lang"]) except ValueError: lang = Language.English - exercise: Exercise = Exercise.query.filter_by(eid=eid).first() + exercise: Exercise = db.session.query(Exercise).filter_by(eid=eid).first() if exercise is None: abort(404) text_field_content: str = "" diff --git a/mc_backend/mcserver/app/models.py b/mc_backend/mcserver/app/models.py index b2402548c7f9a2bc6f95c444a6b902260c93e63c..90fe8cf0646225142d6ed5423c906ba601a15761 100644 --- a/mc_backend/mcserver/app/models.py +++ b/mc_backend/mcserver/app/models.py @@ -1,16 +1,13 @@ """Models for dealing with text data, both in the database and in the application itself.""" -import json -from datetime import datetime from typing import Dict, List, Union, Any - -from flask_restful import fields - from enum import Enum +import typing from sqlalchemy.orm.state import InstanceState from mcserver.app import db from mcserver.config import Config +from mcserver.models_auto import TExercise, Corpus, TCorpus, Exercise, TLearningResult, LearningResult class Case(Enum): @@ -152,150 +149,116 @@ class VocabularyCorpus(Enum): viva = Config.VOCABULARY_VIVA_FILE_NAME -class Corpus(db.Model): - """Model for corpora/texts. Contains metadata for a specific text or corpus.""" - __tablename__ = "Corpus" - - cid = db.Column(db.Integer, primary_key=True) - title = db.Column(db.String(), nullable=False) - source_urn = db.Column(db.String(), unique=True, nullable=False) - uri = db.Column(db.String(), unique=True, nullable=False) - author = db.Column(db.String(), nullable=False) - citation_level_1 = db.Column(db.String(), nullable=False, server_default=CitationLevel.default.value) - citation_level_2 = db.Column(db.String(), nullable=False, server_default=CitationLevel.default.value) - citation_level_3 = db.Column(db.String(), nullable=False, server_default=CitationLevel.default.value) - - def __init__(self, title: str = "", source_urn: str = "", author: str = "", - citation_level_1: Union[CitationLevel, str] = CitationLevel.default, - citation_level_2: Union[CitationLevel, str] = CitationLevel.default, - citation_level_3: Union[CitationLevel, str] = CitationLevel.default, uri: str = "", - json_dict: dict = None): - if json_dict: - self.__dict__ = json_dict - else: - self.title = title - self.source_urn = source_urn - # if no URI was given, this value needs to be changed before another Corpus can be added to the database - self.uri = uri - self.author = author - self.citation_level_1 = citation_level_1 if isinstance(citation_level_1, str) else citation_level_1.value - self.citation_level_2 = citation_level_2 if isinstance(citation_level_2, str) else citation_level_2.value - self.citation_level_3 = citation_level_3 if isinstance(citation_level_3, str) else citation_level_3.value - - def __eq__(self, other): - if isinstance(other, Corpus): - for key in other.__dict__: - if not isinstance(other.__dict__[key], InstanceState) and other.__dict__[key] != self.__dict__[key]: - return False - return True - else: - return False - - -corpus_fields = { - "cid": fields.Integer, - "title": fields.String, - "source_urn": fields.String, - "uri": fields.Url("api.corpus", absolute=False), - "author": fields.String, - "citation_level_1": fields.String, - "citation_level_2": fields.String, - "citation_level_3": fields.String -} - - -class Exercise(db.Model): - """Model for exercises. Holds metadata and content for each exercise in string / JSON format.""" - __tablename__ = "Exercise" - - conll = db.Column(db.String(), nullable=False, server_default="") - correct_feedback = db.Column(db.String()) - eid = db.Column(db.String(), primary_key=True) - exercise_type = db.Column(db.String(), nullable=False) - exercise_type_translation = db.Column(db.String()) - general_feedback = db.Column(db.String()) - incorrect_feedback = db.Column(db.String()) - instructions = db.Column(db.String()) - language = db.Column(db.String(), server_default=Language.German.value) - last_access_time = db.Column(db.DateTime, default=datetime.utcnow, index=True) - partially_correct_feedback = db.Column(db.String()) - search_values = db.Column(db.String(), server_default="") - # solutions are actually List[Solution], but need to be stored as JSON strings in the database - solutions = db.Column(db.String(), nullable=False) - text_complexity = db.Column(db.Float(), server_default="0") - uri = db.Column(db.String(), unique=True, nullable=False) - urn = db.Column(db.String(), nullable=False, server_default="") - work_author = db.Column(db.String(), server_default="") - work_title = db.Column(db.String(), server_default="") - - def __repr__(self): - return "" % self.eid - - def __init__(self, eid: str = "", uri: str = "", exercise_type: str = "", conll: str = "", solutions: str = "", - instructions: str = "", exercise_type_translation: str = "", general_feedback: str = "", - correct_feedback: str = "", partially_correct_feedback: str = "", incorrect_feedback: str = "", - work_author: str = "", last_access_time: datetime = datetime.utcnow(), work_title: str = "", - search_values: str = "", urn: str = "", language: Language = Language.English, - text_complexity: float = 0, json_dict: dict = None): - if json_dict: - self.__dict__ = json_dict - else: - self.conll = conll - self.correct_feedback = correct_feedback - self.eid = eid - self.exercise_type = exercise_type - self.exercise_type_translation = exercise_type_translation - self.general_feedback = general_feedback - self.incorrect_feedback = incorrect_feedback - self.instructions = instructions - self.language = language.value - self.last_access_time = last_access_time - self.partially_correct_feedback = partially_correct_feedback - self.search_values = search_values - self.solutions = solutions - self.text_complexity = text_complexity - self.uri = uri - self.urn = urn - self.work_author = work_author - self.work_title = work_title - - def serialize(self, compress: bool) -> dict: - """ Serializes an exercise to JSON format. """ - ret_val: dict = self.__dict__.copy() - ret_val.pop("_sa_instance_state", None) - ret_val["conll"] = "" if compress else self.conll - # convert Python datetime format to JSON / Javascript, i.e. from seconds to milliseconds - ret_val["last_access_time"] = self.last_access_time.timestamp() * 1000 - ret_val["search_values"] = json.loads(self.search_values) - ret_val["solutions"] = "[]" if compress else json.loads(self.solutions) - return ret_val - +class CorpusMC: + """Keep this synchronized with the implementation in models_auto! + + It is replicated here because the Open-Alchemy package does not correctly assign default values for optional + parameters.""" + + @classmethod + def from_dict(cls, + source_urn: str, + author: str = "Anonymus", + cid: typing.Optional[int] = None, + citation_level_1: str = "default", + citation_level_2: str = "default", + citation_level_3: str = "default", + title: str = "Anonymus", + ) -> TCorpus: + # ignore CID (corpus ID) because it is going to be generated automatically + return Corpus.from_dict( + source_urn=source_urn, author=author, citation_level_1=citation_level_1, + citation_level_2=citation_level_2, citation_level_3=citation_level_3, title=title) + + +class ExerciseMC: + """Keep this synchronized with the implementation in models_auto! + + It is replicated here because the Open-Alchemy package does not correctly assign default values for optional + parameters.""" + + @classmethod + def from_dict(cls, + eid: str, + last_access_time: float, + correct_feedback: str = "", + general_feedback: str = "", + incorrect_feedback: str = "", + instructions: str = "", + partially_correct_feedback: str = "", + search_values: str = "[]", + work_author: str = "", + work_title: str = "", + conll: str = "", + exercise_type: str = "", + exercise_type_translation: str = "", + language: str = "de", + solutions: str = "[]", + text_complexity: float = 0, + urn: str = "", + ) -> TExercise: + return Exercise.from_dict( + eid=eid, last_access_time=last_access_time, correct_feedback=correct_feedback, + general_feedback=general_feedback, incorrect_feedback=incorrect_feedback, + instructions=instructions, partially_correct_feedback=partially_correct_feedback, + search_values=search_values, work_author=work_author, work_title=work_title, + conll=conll, exercise_type=exercise_type, exercise_type_translation=exercise_type_translation, + language=language, solutions=solutions, text_complexity=text_complexity, urn=urn) + + +class LearningResultMC: + """Keep this synchronized with the implementation in models_auto! + + It is replicated here because the Open-Alchemy package does not correctly assign default values for optional + parameters.""" + + @classmethod + def from_dict(cls, + completion: bool, + correct_responses_pattern: str, + created_time: float, + object_definition_description: str, + response: str, + score_max: int, + score_min: int, + score_raw: int, + success: bool, + actor_account_name: str = "", + actor_object_type: str = "", + category_id: str = "", + category_object_type: str = "", + choices: str = "[]", + duration: str = "PT0S", + extensions: str = "{}", + interaction_type: str = "", + object_definition_type: str = "", + object_object_type: str = "", + score_scaled: float = 0, + verb_display: str = "", + verb_id: str = "", + ) -> TLearningResult: + return LearningResult.from_dict( + completion=completion, correct_responses_pattern=correct_responses_pattern, created_time=created_time, + object_definition_description=object_definition_description, response=response, score_max=score_max, + score_min=score_min, score_raw=score_raw, success=success, actor_account_name=actor_account_name, + actor_object_type=actor_object_type, category_id=category_id, category_object_type=category_object_type, + choices=choices, duration=duration, extensions=extensions, interaction_type=interaction_type, + object_definition_type=object_definition_type, object_object_type=object_object_type, + score_scaled=score_scaled, verb_display=verb_display, verb_id=verb_id) -exercise_fields = { - "conll": fields.String, - "eid": fields.String, - "exercise_type": fields.String, - "uri": fields.Url("api.file", absolute=False), - "last_access_time": fields.DateTime(dt_format="iso8601") -} +class Account: + def __init__(self, json_dict: dict): + self.name: str = json_dict["name"] -class UpdateInfo(db.Model): - """Model for updatable entities. Holds information about creation and update time stamps for each resource type.""" - __tablename__ = "UpdateInfo" - - resource_type = db.Column(db.String(), primary_key=True) - created_time = db.Column(db.DateTime, default=datetime.utcnow, index=True) - last_modified_time = db.Column(db.DateTime, default=datetime.fromtimestamp(1), index=True) - def __repr__(self): - return "" % self.resource_type +class Actor: + def __init__(self, json_dict: dict): + self.account: Account = Account(json_dict["account"]) + self.object_type: ObjectType = ObjectType(json_dict["objectType"]) - def __init__(self, resource_type: ResourceType, created_time: datetime = datetime.utcnow(), - last_modified_time: datetime = datetime.utcnow()): - self.resource_type = resource_type.name - self.created_time = created_time - self.last_modified_time = last_modified_time + def serialize(self) -> dict: + return dict(account=self.account.__dict__, objectType=self.object_type.value) class Category: @@ -316,80 +279,6 @@ class Choice: return dict(description={"en-US": self.description.en_us}, id=self.id) -class LearningResult(db.Model): - """Model for learning results. Holds information about exercise type, content and achieved score.""" - __tablename__ = "LearningResult" - - actor_account_name = db.Column(db.String()) - actor_object_type = db.Column(db.String()) - category_id = db.Column(db.String()) - category_object_type = db.Column(db.String()) - choices = db.Column(db.String()) - completion = db.Column(db.Boolean()) - correct_responses_pattern = db.Column(db.String()) - created_time = db.Column(db.DateTime, primary_key=True, index=True) - duration = db.Column(db.String()) - extensions = db.Column(db.String()) - interaction_type = db.Column(db.String()) - object_definition_description = db.Column(db.String()) - object_definition_type = db.Column(db.String()) - object_object_type = db.Column(db.String()) - response = db.Column(db.String()) - score_max = db.Column(db.Integer()) - score_min = db.Column(db.Integer()) - score_raw = db.Column(db.Integer()) - score_scaled = db.Column(db.Float()) - success = db.Column(db.Boolean()) - verb_display = db.Column(db.String()) - verb_id = db.Column(db.String()) - - def __init__(self, actor_account_name: str, actor_object_type: ObjectType, category_id: str, - category_object_type: ObjectType, - choices: List[Choice], completion: bool, - correct_responses_pattern: List[str], created_time: datetime, extensions: Dict[str, object], - duration: str, interaction_type: str, object_definition_description: str, object_definition_type: str, - object_object_type: ObjectType, response: str, score_max: int, score_min: int, score_raw: int, - score_scaled: float, - success: bool, verb_id: str, - verb_display: str): - self.actor_account_name = actor_account_name - self.actor_object_type = actor_object_type.value - self.category_id = category_id - self.category_object_type = category_object_type.value - self.choices = json.dumps([x.serialize() for x in choices]) - self.completion = completion - self.correct_responses_pattern = json.dumps(correct_responses_pattern) - self.created_time = created_time - self.duration = duration - self.extensions = json.dumps(extensions) - self.interaction_type = interaction_type - self.object_definition_description = object_definition_description - self.object_definition_type = object_definition_type - self.object_object_type = object_object_type.value - self.response = response - self.score_max = score_max - self.score_min = score_min - self.score_raw = score_raw - self.score_scaled = score_scaled - self.success = success - self.verb_display = verb_display - self.verb_id = verb_id - - -class Account: - def __init__(self, json_dict: dict): - self.name: str = json_dict["name"] - - -class Actor: - def __init__(self, json_dict: dict): - self.account: Account = Account(json_dict["account"]) - self.object_type: ObjectType = ObjectType(json_dict["objectType"]) - - def serialize(self) -> dict: - return dict(account=self.account.__dict__, objectType=self.object_type.value) - - class Description: def __init__(self, json_dict: dict): self.en_us: str = json_dict["en-US"] diff --git a/mc_backend/mcserver/app/services/corpusService.py b/mc_backend/mcserver/app/services/corpusService.py index 3d4a28285ed8c8d00b3da26f5c4145b59bce3a44..a7b3a669e51c1a08677b28f4dc734c92bc263b30 100644 --- a/mc_backend/mcserver/app/services/corpusService.py +++ b/mc_backend/mcserver/app/services/corpusService.py @@ -15,11 +15,12 @@ from networkx.readwrite import json_graph from requests import HTTPError from mcserver.app import db -from mcserver.app.models import Corpus, CitationLevel, GraphData, Solution, ExerciseType, Phenomenon, FrequencyAnalysis, \ - AnnisResponse, SolutionElement +from mcserver.app.models import CitationLevel, GraphData, Solution, ExerciseType, Phenomenon, FrequencyAnalysis, \ + AnnisResponse, SolutionElement, CorpusMC from mcserver.app.services import AnnotationService, XMLservice, TextService, FileService, FrequencyService, \ CustomCorpusService from mcserver.config import Config +from mcserver.models_auto import Corpus class CorpusService: @@ -41,10 +42,8 @@ class CorpusService: def add_corpus(title_value: str, urn: str, group_name_value: str, citation_levels: List[Union[CitationLevel, str]]) -> None: """Adds a new corpus to the database.""" - new_corpus = Corpus(title=title_value, - source_urn=urn, - author=group_name_value, - citation_level_1=citation_levels[0]) + new_corpus = CorpusMC.from_dict(title=title_value, source_urn=urn, author=group_name_value, + citation_level_1=citation_levels[0]) CorpusService.add_citation_levels(new_corpus, citation_levels) db.session.add(new_corpus) # need to commit once so the Corpus ID (cid) gets generated by the database @@ -338,7 +337,7 @@ class CorpusService: @staticmethod def update_corpora(): """Checks the remote repositories for new corpora to be included in our database.""" - CorpusService.existing_corpora = Corpus.query.all() + CorpusService.existing_corpora = db.session.query(Corpus).all() resolver: HttpCtsRetriever = HttpCtsRetriever(Config.CTS_API_BASE_URL) # check the appropriate literature for the desired author resp: str = resolver.getCapabilities(urn="urn:cts:latinLit") # "urn:cts:greekLit" for Greek @@ -366,14 +365,14 @@ class CorpusService: if urn not in urn_set_existing: CorpusService.add_corpus(title_value, urn, group_name_value, citation_levels) else: - corpus_to_update: Corpus = Corpus.query.filter_by(source_urn=urn).first() + corpus_to_update: Corpus = db.session.query(Corpus).filter_by(source_urn=urn).first() CorpusService.update_corpus(title_value, urn, group_name_value, citation_levels, corpus_to_update) for urn in urn_set_existing: if urn not in urn_set_new: - corpus_to_delete: Corpus = Corpus.query.filter_by(source_urn=urn).first() + corpus_to_delete: Corpus = db.session.query(Corpus).filter_by(source_urn=urn).first() db.session.delete(corpus_to_delete) db.session.commit() - CorpusService.existing_corpora = Corpus.query.all() + CorpusService.existing_corpora = db.session.query(Corpus).all() @staticmethod def update_corpus(title_value: str, urn: str, author: str, diff --git a/mc_backend/mcserver/app/services/customCorpusService.py b/mc_backend/mcserver/app/services/customCorpusService.py index 476a9afd691924d1ced62e7c78fa29c9ce111663..bc7e086086b7013cedddc160935fd6e96be102c9 100644 --- a/mc_backend/mcserver/app/services/customCorpusService.py +++ b/mc_backend/mcserver/app/services/customCorpusService.py @@ -9,62 +9,48 @@ from conllu import TokenList from flask_restful import abort from mcserver import Config -from mcserver.app.models import CustomCorpus, Corpus, CitationLevel, TextPart, Citation +from mcserver.app.models import CustomCorpus, CitationLevel, TextPart, Citation, CorpusMC from mcserver.app.services import AnnotationService, FileService class CustomCorpusService: """ Service for handling access to custom corpora. Performs CRUD-like operations on the database. """ custom_corpora: List[CustomCorpus] = [ - CustomCorpus(corpus=Corpus(title="Commentarii de bello Gallico", - source_urn=Config.CUSTOM_CORPUS_PROIEL_URN_TEMPLATE.format("caes-gal"), - author="C. Iulius Caesar", - citation_level_1=CitationLevel.book, - citation_level_2=CitationLevel.chapter, - citation_level_3=CitationLevel.section), - file_path=Config.CUSTOM_CORPUS_CAES_GAL_FILE_PATH), - CustomCorpus(corpus=Corpus(title="Epistulae ad Atticum", - source_urn=Config.CUSTOM_CORPUS_PROIEL_URN_TEMPLATE.format("cic-att"), - author="M. Tullius Cicero", - citation_level_1=CitationLevel.book, - citation_level_2=CitationLevel.letter, - citation_level_3=CitationLevel.section), - file_path=Config.CUSTOM_CORPUS_CIC_ATT_FILE_PATH), - CustomCorpus(corpus=Corpus(title="De officiis", - source_urn=Config.CUSTOM_CORPUS_PROIEL_URN_TEMPLATE.format("cic-off"), - author="M. Tullius Cicero", - citation_level_1=CitationLevel.book, - citation_level_2=CitationLevel.section, - citation_level_3=CitationLevel.default), - file_path=Config.CUSTOM_CORPUS_CIC_OFF_FILE_PATH), - CustomCorpus(corpus=Corpus(title="Vulgata (Novum Testamentum)", - source_urn=Config.CUSTOM_CORPUS_PROIEL_URN_TEMPLATE.format("latin-nt"), - author="Hieronymus", - citation_level_1=CitationLevel.chapter, - citation_level_2=CitationLevel.section, - citation_level_3=CitationLevel.default), - file_path=Config.CUSTOM_CORPUS_LATIN_NT_FILE_PATH), - CustomCorpus(corpus=Corpus(title="Opus Agriculturae", - source_urn=Config.CUSTOM_CORPUS_PROIEL_URN_TEMPLATE.format("pal-agr"), - author="Palladius", - citation_level_1=CitationLevel.book, - citation_level_2=CitationLevel.chapter, - citation_level_3=CitationLevel.section), - file_path=Config.CUSTOM_CORPUS_PAL_AGR_FILE_PATH), - CustomCorpus(corpus=Corpus(title="Peregrinatio Aetheriae", - source_urn=Config.CUSTOM_CORPUS_PROIEL_URN_TEMPLATE.format("per-aeth"), - author="Peregrinatio Aetheriae", - citation_level_1=CitationLevel.chapter, - citation_level_2=CitationLevel.section, - citation_level_3=CitationLevel.default), - file_path=Config.CUSTOM_CORPUS_PER_AET_FILE_PATH), - CustomCorpus(corpus=Corpus(title="VIVA", - source_urn=Config.CUSTOM_CORPUS_VIVA_URN, - author="VIVA (textbook)", - citation_level_1=CitationLevel.book, - citation_level_2=CitationLevel.unit, - citation_level_3=CitationLevel.default), - file_path=Config.CUSTOM_CORPUS_VIVA_FILE_PATH)] + CustomCorpus(corpus=CorpusMC.from_dict( + title="Commentarii de bello Gallico", source_urn=Config.CUSTOM_CORPUS_PROIEL_URN_TEMPLATE.format( + "caes-gal"), author="C. Iulius Caesar", citation_level_1=CitationLevel.book.value, + citation_level_2=CitationLevel.chapter.value, citation_level_3=CitationLevel.section.value), + file_path=Config.CUSTOM_CORPUS_CAES_GAL_FILE_PATH), + CustomCorpus(corpus=CorpusMC.from_dict( + title="Epistulae ad Atticum", source_urn=Config.CUSTOM_CORPUS_PROIEL_URN_TEMPLATE.format( + "cic-att"), author="M. Tullius Cicero", citation_level_1=CitationLevel.book.value, + citation_level_2=CitationLevel.letter.value, citation_level_3=CitationLevel.section.value), + file_path=Config.CUSTOM_CORPUS_CIC_ATT_FILE_PATH), + CustomCorpus(corpus=CorpusMC.from_dict( + title="De officiis", source_urn=Config.CUSTOM_CORPUS_PROIEL_URN_TEMPLATE.format( + "cic-off"), author="M. Tullius Cicero", citation_level_1=CitationLevel.book.value, + citation_level_2=CitationLevel.section.value, citation_level_3=CitationLevel.default.value), + file_path=Config.CUSTOM_CORPUS_CIC_OFF_FILE_PATH), + CustomCorpus(corpus=CorpusMC.from_dict( + title="Vulgata (Novum Testamentum)", source_urn=Config.CUSTOM_CORPUS_PROIEL_URN_TEMPLATE.format( + "latin-nt"), author="Hieronymus", citation_level_1=CitationLevel.chapter.value, + citation_level_2=CitationLevel.section.value, citation_level_3=CitationLevel.default.value), + file_path=Config.CUSTOM_CORPUS_LATIN_NT_FILE_PATH), + CustomCorpus(corpus=CorpusMC.from_dict( + title="Opus Agriculturae", source_urn=Config.CUSTOM_CORPUS_PROIEL_URN_TEMPLATE.format( + "pal-agr"), author="Palladius", citation_level_1=CitationLevel.book.value, + citation_level_2=CitationLevel.chapter.value, citation_level_3=CitationLevel.section.value), + file_path=Config.CUSTOM_CORPUS_PAL_AGR_FILE_PATH), + CustomCorpus(corpus=CorpusMC.from_dict( + title="Peregrinatio Aetheriae", source_urn=Config.CUSTOM_CORPUS_PROIEL_URN_TEMPLATE.format( + "per-aeth"), author="Peregrinatio Aetheriae", citation_level_1=CitationLevel.chapter.value, + citation_level_2=CitationLevel.section.value, citation_level_3=CitationLevel.default.value), + file_path=Config.CUSTOM_CORPUS_PER_AET_FILE_PATH), + CustomCorpus(corpus=CorpusMC.from_dict( + title="VIVA", source_urn=Config.CUSTOM_CORPUS_VIVA_URN, author="VIVA (textbook)", + citation_level_1=CitationLevel.book.value, citation_level_2=CitationLevel.unit.value, + citation_level_3=CitationLevel.default.value), file_path=Config.CUSTOM_CORPUS_VIVA_FILE_PATH) + ] makra_map: Dict[str, str] = {"ā": "a", "Ā": "A", "ē": "e", "Ē": "E", "ō": "o", "Ō": "O", "ī": "i", "Ī": "i", "ū": "u"} punctuation_extended_rare: str = "≫≪–›‹»«…" diff --git a/mc_backend/mcserver/app/services/databaseService.py b/mc_backend/mcserver/app/services/databaseService.py index e397f9f07d842f9c74f97521d0975c062d0cb1c6..f197de1614b411a5d7ce4aee9dcf6c2de221a843 100644 --- a/mc_backend/mcserver/app/services/databaseService.py +++ b/mc_backend/mcserver/app/services/databaseService.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import List, Dict +from typing import List, Dict, Any from flask import Flask from flask_migrate import stamp, upgrade @@ -7,10 +7,11 @@ import rapidjson as json from sqlalchemy.exc import OperationalError from mcserver.app import db -from mcserver.app.models import UpdateInfo, Corpus, CitationLevel, Exercise, ResourceType, TextComplexityMeasure, \ +from mcserver.app.models import CitationLevel, ResourceType, TextComplexityMeasure, \ AnnisResponse, GraphData, TextComplexity from mcserver.app.services import CorpusService, CustomCorpusService, TextComplexityService from mcserver.config import Config +from mcserver.models_auto import Corpus, Exercise, TExercise, UpdateInfo class DatabaseService: @@ -18,15 +19,17 @@ class DatabaseService: @staticmethod def check_corpus_list_age(app: Flask) -> None: """ Checks whether the corpus list needs to be updated. If yes, it performs the update. """ - ui_cts: UpdateInfo = UpdateInfo.query.filter_by(resource_type=ResourceType.cts_data.name).first() + ui_cts: UpdateInfo = db.session.query(UpdateInfo).filter_by(resource_type=ResourceType.cts_data.name).first() if ui_cts is None: return - elif (datetime.utcnow() - ui_cts.last_modified_time).total_seconds() > Config.INTERVAL_CORPUS_UPDATE: - app.logger.info("Corpus update started.") - CorpusService.update_corpora() - ui_cts.last_modified_time = datetime.utcnow() - db.session.commit() - app.logger.info("Corpus update completed.") + else: + ui_datetime: datetime = datetime.fromtimestamp(ui_cts.last_modified_time) + if (datetime.utcnow() - ui_datetime).total_seconds() > Config.INTERVAL_CORPUS_UPDATE: + app.logger.info("Corpus update started.") + CorpusService.update_corpora() + ui_cts.last_modified_time = datetime.utcnow().timestamp() + db.session.commit() + app.logger.info("Corpus update completed.") @staticmethod def init_db_alembic() -> None: @@ -39,7 +42,7 @@ class DatabaseService: def init_db_corpus() -> None: """Initializes the corpus list if it is not already there and up to date.""" if db.engine.dialect.has_table(db.engine, "Corpus"): - CorpusService.existing_corpora = Corpus.query.all() + CorpusService.existing_corpora = db.session.query(Corpus).all() urn_dict: Dict[str, int] = {v.source_urn: i for i, v in enumerate(CorpusService.existing_corpora)} for cc in CustomCorpusService.custom_corpora: if cc.corpus.source_urn in urn_dict: @@ -55,17 +58,17 @@ class DatabaseService: CorpusService.add_corpus(title_value=cc.corpus.title, urn=cc.corpus.source_urn, group_name_value=cc.corpus.author, citation_levels=citation_levels) - CorpusService.existing_corpora = Corpus.query.all() + CorpusService.existing_corpora = db.session.query(Corpus).all() @staticmethod def init_db_update_info() -> None: """Initializes update entries for all resources that have not yet been created.""" if db.engine.dialect.has_table(db.engine, "UpdateInfo"): for rt in ResourceType: - ui_cts: UpdateInfo = UpdateInfo.query.filter_by(resource_type=rt.name).first() + ui_cts: UpdateInfo = db.session.query(UpdateInfo).filter_by(resource_type=rt.name).first() if ui_cts is None: - ui_cts = UpdateInfo(resource_type=rt, last_modified_time=datetime.utcfromtimestamp(1), - created_time=datetime.utcnow()) + ui_cts = UpdateInfo.from_dict(resource_type=rt.name, last_modified_time=1, + created_time=datetime.utcnow().timestamp()) db.session.add(ui_cts) db.session.commit() @@ -88,12 +91,13 @@ class DatabaseService: def update_exercises(is_csm: bool) -> None: """Deletes old exercises.""" if db.engine.dialect.has_table(db.engine, "Exercise"): - exercises: List[Exercise] = Exercise.query.all() + exercises: List[Exercise] = db.session.query(Exercise).all() now: datetime = datetime.utcnow() for exercise in exercises: + exercise_datetime: datetime = datetime.fromtimestamp(exercise.last_access_time) # delete exercises that have not been accessed for a while, are not compatible anymore, or contain # corrupted / empty data - if (now - exercise.last_access_time).total_seconds() > Config.INTERVAL_EXERCISE_DELETE or \ + if (now - exercise_datetime).total_seconds() > Config.INTERVAL_EXERCISE_DELETE or \ not exercise.urn or not json.loads(exercise.solutions): db.session.delete(exercise) db.session.commit() diff --git a/mc_backend/mcserver/app/services/fileService.py b/mc_backend/mcserver/app/services/fileService.py index 9ce0cd0a9183af6208e0682036021fdf08b5289f..e4c7b9ef2c5c57de750cfcb6f56d7c52a5fc5dcf 100644 --- a/mc_backend/mcserver/app/services/fileService.py +++ b/mc_backend/mcserver/app/services/fileService.py @@ -13,9 +13,10 @@ from docx.text.run import Run from xhtml2pdf import pisa from mcserver import Config -from mcserver.app.models import DownloadableFile, FileType, Solution, Exercise, ExerciseType, SolutionElement, \ +from mcserver.app.models import DownloadableFile, FileType, Solution, ExerciseType, SolutionElement, \ VocabularyCorpus from mcserver.app.services import TextService, XMLservice +from mcserver.models_auto import Exercise class FileService: diff --git a/mc_backend/mcserver/app/services/networkService.py b/mc_backend/mcserver/app/services/networkService.py index b26a40d81541b326bd2ec3542e5a25307d5414ed..b6c719950fb29b983d2c1be70b0a3c4ecb04f5c4 100644 --- a/mc_backend/mcserver/app/services/networkService.py +++ b/mc_backend/mcserver/app/services/networkService.py @@ -1,3 +1,4 @@ +import json from datetime import datetime from typing import Dict @@ -6,7 +7,9 @@ import rapidjson from flask import Response from flask_restful.reqparse import RequestParser +from mcserver import Config from mcserver.app.models import StaticExercise +from mcserver.models_auto import Exercise, TExercise class NetworkService: @@ -15,12 +18,27 @@ class NetworkService: exercises_last_update: datetime = datetime.fromtimestamp(0) @staticmethod - def make_json_response(response_input: object): + def get_exercise_uri(exercise: Exercise): + return f"{Config.SERVER_URI_FILE}/{exercise.eid}" + + @staticmethod + def make_json_response(response_input: object, indent: int = None) -> Response: """Transforms the resulting objects to JSON so we can send them to the client.""" - dump_result = rapidjson.dumps(response_input) + dump_result = rapidjson.dumps(response_input, indent=indent) response: Response = Response(dump_result, mimetype="application/json") # prevent CORS (double check in addition to using the Flask-CORS module) response.headers.add('Access-Control-Allow-Origin', '*') # prevent CORB response.headers.add('Access-Control-Allow-Headers', "Content-Type") return response + + @staticmethod + def serialize_exercise(exercise: TExercise, compress: bool) -> dict: + """ Serializes an exercise to JSON format. """ + ret_val: dict = exercise.to_dict() + ret_val["conll"] = "" if compress else exercise.conll + # convert the POSIX timestamp to JSON / Javascript, i.e. from seconds to milliseconds + ret_val["last_access_time"] = exercise.last_access_time * 1000 + ret_val["search_values"] = json.loads(exercise.search_values) + ret_val["solutions"] = "[]" if compress else json.loads(exercise.solutions) + return ret_val diff --git a/mc_backend/mcserver/app/services/textComplexityService.py b/mc_backend/mcserver/app/services/textComplexityService.py index d27cd7519fee6e9117e722d98f8b9c169c01486f..de3a2cbe025a814a9a0a8ad58ce0b24716aa0874 100644 --- a/mc_backend/mcserver/app/services/textComplexityService.py +++ b/mc_backend/mcserver/app/services/textComplexityService.py @@ -48,11 +48,12 @@ class TextComplexityService: tc_measure_overall.append(tc.n_types / tc.n_w * 100) tc_measure_overall.append(tc.lex_den * 100) # all the other measures need to be normalized for text length, e.g. word/sentence/clause count - tc_measure_overall.append((tc.n_subclause / (tc.n_clause + tc.n_subclause)) * 100) - tc_measure_overall.append((tc.n_abl_abs / (tc.n_clause + tc.n_subclause)) * 100) - tc_measure_overall.append((tc.n_gerund / (tc.n_clause + tc.n_subclause)) * 100) - tc_measure_overall.append((tc.n_inf / (tc.n_clause + tc.n_subclause)) * 100) - tc_measure_overall.append((tc.n_part / (tc.n_clause + tc.n_subclause)) * 100) + divisor: int = max(tc.n_clause + tc.n_subclause, 1) + tc_measure_overall.append((tc.n_subclause / divisor) * 100) + tc_measure_overall.append((tc.n_abl_abs / divisor) * 100) + tc_measure_overall.append((tc.n_gerund / divisor) * 100) + tc_measure_overall.append((tc.n_inf / divisor) * 100) + tc_measure_overall.append((tc.n_part / divisor) * 100) return round(sum(tc_measure_overall) / len(tc_measure_overall), 2) @staticmethod diff --git a/mc_backend/mcserver/app/services/textService.py b/mc_backend/mcserver/app/services/textService.py index c7c388396ae72b43a5331377c135191e7d734773..97cd513f19b064e91417a46a37dfbd4c0daa1982 100644 --- a/mc_backend/mcserver/app/services/textService.py +++ b/mc_backend/mcserver/app/services/textService.py @@ -10,8 +10,9 @@ import conllu import requests from conllu import TokenList -from mcserver.app.models import Exercise, Solution +from mcserver.app.models import Solution from mcserver.config import Config +from mcserver.models_auto import Exercise class TextService: diff --git a/mc_backend/mcserver/app/services/xmlService.py b/mc_backend/mcserver/app/services/xmlService.py index 79f51074e67b8aa1698e1abb59f85b0c13eb35a8..8a2fba23a225200363aa89b0e08fb12c9478a865 100644 --- a/mc_backend/mcserver/app/services/xmlService.py +++ b/mc_backend/mcserver/app/services/xmlService.py @@ -6,8 +6,9 @@ from lxml import etree, objectify from lxml.etree import _ElementUnicodeResult from collections import OrderedDict -from mcserver.app.models import ExerciseType, Exercise, FileType, Solution +from mcserver.app.models import ExerciseType, FileType, Solution from mcserver.app.services import TextService +from mcserver.models_auto import Exercise class XMLservice: diff --git a/mc_backend/mcserver/config.py b/mc_backend/mcserver/config.py index b942a2e849e43930b090689225a6a63c28d41b0b..3837e318fc786f79d7e4bd48f604d48f46fe8959 100644 --- a/mc_backend/mcserver/config.py +++ b/mc_backend/mcserver/config.py @@ -33,6 +33,7 @@ class Config(object): TREEBANKS_PATH = os.path.join(ASSETS_DIRECTORY, "treebanks") TREEBANKS_PROIEL_PATH = os.path.join(TREEBANKS_PATH, "proiel") + API_SPEC_FILE_PATH = os.path.join(MC_SERVER_DIRECTORY, "mcserver_api.yaml") AQL_CASE = "/.*Case=.*/" AQL_DEP = "->dep" AQL_DEPREL = "deprel" @@ -57,12 +58,21 @@ class Config(object): CUSTOM_CORPUS_VIVA_URN = "urn:custom:latinLit:viva.lat" CUSTOM_CORPUS_PROIEL_URN_TEMPLATE = "urn:custom:latinLit:proiel.{0}.lat" DATABASE_TABLE_ALEMBIC = "alembic_version" + DATABASE_URL_DOCKER = "postgresql://postgres@db:5432/" DATABASE_URL_LOCAL = "postgresql://postgres@0.0.0.0:5432/postgres" - DATABASE_URL_FALLBACK = "postgresql://postgres@db:5432/" if IS_DOCKER else DATABASE_URL_LOCAL + DATABASE_URL_SQLITE = f"sqlite:///{os.path.join(basedir, 'app.db')}" + DATABASE_URL_SQLITE_MEMORY = "sqlite:///:memory:" + DATABASE_URL_FALLBACK = DATABASE_URL_DOCKER if IS_DOCKER else DATABASE_URL_SQLITE DATABASE_URL = os.environ.get("DATABASE_URL", DATABASE_URL_FALLBACK) DEBUG = False DOCKER_SERVICE_NAME_CSM = "csm" DOCKER_SERVICE_NAME_MCSERVER = "mcserver" + ERROR_MESSAGE_CORPUS_NOT_FOUND = "A corpus with the specified ID was not found!" + ERROR_MESSAGE_EXERCISE_NOT_FOUND = "An exercise with the specified ID was not found!" + ERROR_MESSAGE_INTERNAL_SERVER_ERROR = "The server encountered an unexpected condition that prevented it from " \ + "fulfilling the request." + ERROR_TITLE_INTERNAL_SERVER_ERROR = "Internal Server Error" + ERROR_TITLE_NOT_FOUND = "Not found" FAVICON_FILE_NAME = "favicon.ico" FLASK_MIGRATE = "migrate" GRAPHANNIS_DEPENDENCY_LINK = "dep" @@ -114,7 +124,7 @@ class Config(object): SERVER_URI_VECTOR_NETWORK = SERVER_URI_BASE + "vectorNetwork" SERVER_URI_VOCABULARY = SERVER_URI_BASE + "vocabulary" # END endpoints - SQLALCHEMY_DATABASE_URI = os.environ.get("DATABASE_URL") or "sqlite:///" + os.path.join(basedir, "app.db") + SQLALCHEMY_DATABASE_URI = os.environ.get("DATABASE_URL") or DATABASE_URL_SQLITE SQLALCHEMY_TRACK_MODIFICATIONS = False STATIC_EXERCISES_REPOSITORY_URL = "https://scm.cms.hu-berlin.de/callidus/machina-callida/-/archive/master/machina-callida-master.zip?path=mc_frontend%2Fsrc%2Fassets%2Fh5p" STOP_WORDS_LATIN_PATH = os.path.join(CACHE_DIRECTORY, "stop_words_latin.json") @@ -149,7 +159,8 @@ class DevelopmentConfig(Config): """Configuration for the development environment""" DEVELOPMENT = True DEBUG = True - SQLALCHEMY_DATABASE_URI = os.environ.get("DATABASE_URL", Config.DATABASE_URL_FALLBACK) + SQLALCHEMY_DATABASE_URI = Config.DATABASE_URL_DOCKER if Config.IS_DOCKER else \ + os.environ.get("DATABASE_URL", Config.DATABASE_URL_FALLBACK) class TestingConfig(Config): @@ -162,7 +173,7 @@ class TestingConfig(Config): SIMULATE_CORPUS_NOT_FOUND = False SIMULATE_EMPTY_GRAPH = False SIMULATE_HTTP_ERROR = False - SQLALCHEMY_DATABASE_URI = f"sqlite:///{os.path.join(basedir, 'app.db')}" + SQLALCHEMY_DATABASE_URI = Config.DATABASE_URL_SQLITE STATIC_EXERCISES_ZIP_FILE_PATH = os.path.join(Config.TMP_DIRECTORY, "static_exercises.zip") TESTING = True diff --git a/mc_backend/mcserver/mcserver_api.yaml b/mc_backend/mcserver/mcserver_api.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6179a6935c799c6c0154cc36d336f70f5ba2237d --- /dev/null +++ b/mc_backend/mcserver/mcserver_api.yaml @@ -0,0 +1,444 @@ +openapi: "3.0.0" + +info: + title: Machina Callida Backend REST API + version: "1.0" +servers: + - url: http://localhost:5000/mc/api/v1.0 + +paths: + /corpora: + get: + summary: Returns a list of corpora. + operationId: mcserver.app.api.corpusListAPI.get + responses: + 200: + description: Corpus list + content: + application/json: + schema: + $ref: '#/components/schemas/Corpus' + parameters: + - name: last_update_time + in: query + description: Time (in milliseconds) of the last update. + required: true + schema: + type: integer + example: 123456789 + /corpora/{cid}: + parameters: + - name: cid + in: path + description: Corpus identifier. + required: true + schema: + type: integer + example: 1 + delete: + summary: Deletes a single corpus by ID. + operationId: mcserver.app.api.corpusAPI.delete + responses: + 200: + description: Indication of success + content: + application/json: + schema: + type: boolean + example: true + get: + summary: Returns a single corpus by ID. + operationId: mcserver.app.api.corpusAPI.get + responses: + 200: + description: Corpus object + content: + application/json: + schema: + $ref: '#/components/schemas/Corpus' + patch: + summary: Updates a single corpus by ID. + operationId: mcserver.app.api.corpusAPI.patch + responses: + 200: + description: Corpus object + content: + application/json: + schema: + $ref: '#/components/schemas/Corpus' + parameters: + - name: author + in: query + description: Author of the texts in the corpus. + required: false + schema: + type: string + example: Aulus Gellius + - name: source_urn + in: query + description: CTS base URN for referencing the corpus. + required: false + schema: + type: string + example: urn:cts:latinLit:phi1254.phi001.perseus-lat2 + - name: title + in: query + description: Corpus title. + required: false + schema: + type: string + example: Noctes Atticae + /exercise: + get: + summary: Returns exercise data by ID. + operationId: mcserver.app.api.exerciseAPI.get + responses: + 200: + description: Exercise data object + # TODO: SPECIFY RESPONSE SCHEMA + parameters: + - name: eid + in: query + description: Unique identifier (UUID) for the exercise. + required: true + schema: + type: string + example: 12345678-1234-5678-1234-567812345678 + post: + summary: Creates a new exercise. + operationId: mcserver.app.api.exerciseAPI.post + responses: + 200: + description: Exercise data object + # TODO: SPECIFY RESPONSE SCHEMA + requestBody: + $ref: '#/components/requestBodies/ExerciseForm' +components: + requestBodies: + ExerciseForm: + required: true + content: + application/x-www-form-urlencoded: + schema: + x-body-name: exercise_data + type: object + allOf: + - $ref: '#/components/schemas/ExerciseBase' + required: + - instructions + - search_values + - description: Additional exercise data. + type: object + properties: + type: + type: string + description: Type of exercise, concerning interaction and layout. + example: markWords + type_translation: + type: string + description: Localized expression of the exercise type. + example: Cloze + urn: + type: string + description: CTS URN for the text passage from which the exercise was created. + example: urn:cts:latinLit:phi0448.phi001.perseus-lat2:1.1.1 + required: + - type + schemas: + Corpus: # Object definition + description: Collection of texts. + type: object # Data type + x-tablename: Corpus + properties: + author: + type: string + description: Author of the texts in the corpus. + example: Aulus Gellius + default: "Anonymus" + nullable: false + cid: + type: integer + description: Unique identifier for the corpus. + example: 1 + x-primary-key: true + x-autoincrement: true + citation_level_1: + type: string + description: First level for citing the corpus. + example: Book + default: default + citation_level_2: + type: string + description: Second level for citing the corpus. + example: Chapter + default: default + citation_level_3: + type: string + description: Third level for citing the corpus. + example: Section + default: default + source_urn: + type: string + description: CTS base URN for referencing the corpus. + example: urn:cts:latinLit:phi1254.phi001.perseus-lat2 + x-unique: true + title: + type: string + description: Corpus title. + example: Noctes Atticae + nullable: false + default: Anonymus + required: + - source_urn + Exercise: + allOf: + - $ref: "#/components/schemas/ExerciseBase" + - description: Data for creating and evaluating interactive exercises. + type: object # Data type + x-tablename: Exercise + properties: + conll: + type: string + description: CONLL-formatted linguistic annotations represented as a single string. + example: \# newdoc id = ...\n# sent_id = 1\n# text = Caesar fortis est.\n1\tCaesar\tCaeso\tVERB ... + default: "" + nullable: false + eid: + type: string + description: Unique identifier (UUID) for the exercise. + example: 12345678-1234-5678-1234-567812345678 + x-primary-key: true + exercise_type: + type: string + description: Type of exercise, concerning interaction and layout. + example: markWords + default: "" + nullable: false + exercise_type_translation: + type: string + description: Localized expression of the exercise type. + example: Cloze + default: "" + language: + type: string + description: ISO 639-1 Language Code for the localization of exercise content. + example: en + default: de + last_access_time: + type: number + format: float + description: When the exercise was last accessed (as POSIX timestamp). + example: 1234567.789 + x-index: true + solutions: + type: string + description: Correct solutions for the exercise. + example: "[{'target': {'sentence_id': 1, 'token_id': 7, 'salt_id': 'salt:/urn:...', 'content': 'eo'}, 'value': {'sentence_id': 0, 'token_id': 0, 'content': None, 'salt_id': 'salt:/urn:...'}}]" + default: "[]" + nullable: false + text_complexity: + type: number + format: float + description: Overall text complexity as measured by the software's internal language analysis. + example: 54.53 + default: 0 + urn: + type: string + description: CTS URN for the text passage from which the exercise was created. + example: urn:cts:latinLit:phi0448.phi001.perseus-lat2:1.1.1 + default: "" + nullable: false + required: + - eid + - last_access_time + LearningResult: + description: Learner data for completed exercises. + type: object + x-tablename: LearningResult + properties: + actor_account_name: + type: string + description: H5P user ID, usually unique per device. + example: ebea3f3e-7410-4215-b34d-c1417f7c7c18 + default: "" + actor_object_type: + type: string + description: Describes the kind of object that was recognized as actor. + example: Agent + default: "" + category_id: + type: string + description: Link to the exercise type specification. + example: http://h5p.org/libraries/H5P.MarkTheWords-1.9 + default: "" + category_object_type: + type: string + description: Describes the kind of object that was recognized as exercise. + example: Activity + default: "" + choices: + type: string + description: JSON string containing a list of possible choices, each with ID and description. + example: "[{'id':'2','description':{'en-US':'Quintus ist bei allen in der Provinz beliebt.\n'}},{'id':'3','description':{'en-US':'Asia ist eine unbekannte Provinz.\n'}}]" + default: "[]" + completion: + type: boolean + description: Whether the exercise was fully processed or not. + example: true + correct_responses_pattern: + type: string + description: JSON string containing a list of possible solutions to the exercise, given as patterns of answers. + example: "['0[,]1[,]2']" + created_time: + type: number + format: float + description: When the learner data was received (POSIX timestamp). + example: 1234567.789 + x-index: true + x-primary-key: true + duration: + type: string + description: How many seconds it took a learner to complete the exercise. + example: PT9.19S + default: "PT0S" + extensions: + type: string + description: JSON string containing a mapping of keys and values (usually the local content ID, i.e. a versioning mechanism). + example: "{'http://h5p.org/x-api/h5p-local-content-id':1}" + default: "{}" + interaction_type: + type: string + description: Exercise type. + example: choice + default: "" + object_definition_description: + type: string + description: Exercise content, possibly including instructions. + example: "Bestimme die Form von custodem im Satz: Urbs custodem non tyrannum, domus hospitem non expilatorem recepit.\n" + object_definition_type: + type: string + description: Type of object definition that is presented to the user. + example: http://adlnet.gov/expapi/activities/cmi.interaction + default: "" + object_object_type: + type: string + description: Type of object that is presented to the user. + example: Activity + default: "" + response: + type: string + description: Answer provided by the user, possibly as a pattern. + example: His in rebus[,]sociis[,]civibus[,]rei publicae + score_max: + type: integer + description: Maximum possible score to be achieved in this exercise. + example: 1 + score_min: + type: integer + description: Minimum score to be achieved in this exercise. + example: 0 + score_raw: + type: integer + description: Score that was actually achieved by the user in this exercise. + example: 1 + score_scaled: + type: number + format: float + description: Relative score (between 0 and 1) that was actually achieved by the user in this exercise. + example: 0.8889 + default: 0 + success: + type: boolean + description: Whether the exercise was successfully completed or not. + example: true + verb_display: + type: string + description: Type of action that was performed by the user. + example: answered + default: "" + verb_id: + type: string + description: Link to the type of action that was performed by the user. + example: http://adlnet.gov/expapi/verbs/answered + default: "" + required: + - completion + - correct_responses_pattern + - created_time + - object_definition_description + - response + - score_max + - score_min + - score_raw + - success + UpdateInfo: + description: Timestamps for updates of various resources. + type: object + x-tablename: UpdateInfo + properties: + created_time: + type: number + format: float + description: When the resource was created (as POSIX timestamp). + example: 1234567.789 + x-index: true + last_modified_time: + type: number + format: float + description: When the resource was last modified (as POSIX timestamp). + example: 1234567.789 + x-index: true + resource_type: + type: string + enum: [cts_data, exercise_list, file_api_clean] + description: Name of the resource for which update timestamps are indexed. + example: cts_data + x-primary-key: true + required: + - created_time + - last_modified_time + - resource_type + ExerciseBase: + description: Base data for creating and evaluating interactive exercises. + type: object + properties: + correct_feedback: + type: string + description: Feedback for successful completion of the exercise. + example: Well done! + default: "" + general_feedback: + type: string + description: Feedback for finishing the exercise. + example: You have finished the exercise. + default: "" + incorrect_feedback: + type: string + description: Feedback for failing to complete the exercise successfully. + example: Unfortunately, that answer is wrong. + default: "" + instructions: + type: string + description: Hints for how to complete the exercise. + example: Fill in the gaps! + default: "" + partially_correct_feedback: + type: string + description: Feedback for successfully completing certain parts of the exercise. + example: Some parts of this answer are correct. + default: "" + search_values: + type: string + description: Search queries that were used to build the exercise. + example: "['upostag=noun', 'dependency=object']" + default: "[]" + work_author: + type: string + description: Name of the person who wrote the base text for the exercise. + example: C. Iulius Caesar + default: "" + work_title: + type: string + description: Title of the base text for the exercise. + example: Noctes Atticae + default: "" diff --git a/mc_backend/mcserver/migrations/alembic.ini b/mc_backend/mcserver/migrations/alembic.ini index 405cd14a1f518db6441189428f2d811de9af13bc..b50819209d63149851f984802316b88d9348421b 100644 --- a/mc_backend/mcserver/migrations/alembic.ini +++ b/mc_backend/mcserver/migrations/alembic.ini @@ -8,7 +8,6 @@ # the 'revision' command, regardless of autogenerate # revision_environment = false - # Logging configuration [loggers] keys = root,sqlalchemy,alembic diff --git a/mc_backend/mcserver/migrations/env.py b/mc_backend/mcserver/migrations/env.py index 23663ff2f54e6c4425953537976b175246c8a9e6..24346e3c2890c12ae6b68b9fcad5294b8242e2a5 100644 --- a/mc_backend/mcserver/migrations/env.py +++ b/mc_backend/mcserver/migrations/env.py @@ -3,7 +3,9 @@ from alembic import context from sqlalchemy import engine_from_config, pool from logging.config import fileConfig import logging +import open_alchemy +open_alchemy.init_yaml(spec_filename="mcserver/mcserver_api.yaml") # this is the Alembic Config object, which provides # access to the values within the .ini file in use. config = context.config @@ -18,9 +20,12 @@ logger = logging.getLogger('alembic.env') # from myapp import mymodel # target_metadata = mymodel.Base.metadata from flask import current_app + config.set_main_option('sqlalchemy.url', current_app.config.get('SQLALCHEMY_DATABASE_URI')) -target_metadata = current_app.extensions['migrate'].db.metadata +# target_metadata = current_app.extensions['migrate'].db.metadata +target_metadata = open_alchemy.models.Base.metadata + # other values from the config, defined by the needs of env.py, # can be acquired: @@ -81,6 +86,7 @@ def run_migrations_online(): finally: connection.close() + if context.is_offline_mode(): run_migrations_offline() else: diff --git a/mc_backend/mcserver/migrations/versions/9b7d54dd5411_.py b/mc_backend/mcserver/migrations/versions/9b7d54dd5411_.py new file mode 100644 index 0000000000000000000000000000000000000000..66288d6fecb76bf424c6003f0f5bf3e4e72ec23e --- /dev/null +++ b/mc_backend/mcserver/migrations/versions/9b7d54dd5411_.py @@ -0,0 +1,263 @@ +"""empty message + +Revision ID: 9b7d54dd5411 +Revises: 905858a5c465 +Create Date: 2020-05-15 15:07:47.573520 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = '9b7d54dd5411' +down_revision = '905858a5c465' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint('Corpus_uri_key', 'Corpus', type_='unique') + op.drop_column('Corpus', 'uri') + op.alter_column('Exercise', 'correct_feedback', + existing_type=sa.VARCHAR(), + nullable=False) + op.alter_column('Exercise', 'exercise_type_translation', + existing_type=sa.VARCHAR(), + nullable=False) + op.alter_column('Exercise', 'general_feedback', + existing_type=sa.VARCHAR(), + nullable=False) + op.alter_column('Exercise', 'incorrect_feedback', + existing_type=sa.VARCHAR(), + nullable=False) + op.alter_column('Exercise', 'instructions', + existing_type=sa.VARCHAR(), + nullable=False) + op.alter_column('Exercise', 'language', + existing_type=sa.VARCHAR(), + nullable=False, + existing_server_default=sa.text("'de'::character varying")) + op.alter_column('Exercise', 'last_access_time', + existing_type=postgresql.TIMESTAMP(), type_=sa.FLOAT(), + postgresql_using="date_part('epoch',last_access_time)::float", nullable=False) + op.alter_column('Exercise', 'partially_correct_feedback', + existing_type=sa.VARCHAR(), + nullable=False) + op.alter_column('Exercise', 'search_values', + existing_type=sa.VARCHAR(), + nullable=False, + existing_server_default=sa.text("''::character varying")) + op.alter_column('Exercise', 'text_complexity', + existing_type=postgresql.DOUBLE_PRECISION(precision=53), + nullable=False, + existing_server_default=sa.text("'0'::double precision")) + op.alter_column('Exercise', 'work_author', + existing_type=sa.VARCHAR(), + nullable=False, + existing_server_default=sa.text("''::character varying")) + op.alter_column('Exercise', 'work_title', + existing_type=sa.VARCHAR(), + nullable=False, + existing_server_default=sa.text("''::character varying")) + op.drop_constraint('Exercise_uri_key', 'Exercise', type_='unique') + op.drop_column('Exercise', 'uri') + op.alter_column('LearningResult', 'actor_account_name', + existing_type=sa.VARCHAR(), + nullable=False) + op.alter_column('LearningResult', 'actor_object_type', + existing_type=sa.VARCHAR(), + nullable=False) + op.alter_column('LearningResult', 'category_id', + existing_type=sa.VARCHAR(), + nullable=False) + op.alter_column('LearningResult', 'category_object_type', + existing_type=sa.VARCHAR(), + nullable=False) + op.alter_column('LearningResult', 'choices', + existing_type=sa.VARCHAR(), + nullable=False) + op.alter_column('LearningResult', 'completion', + existing_type=sa.BOOLEAN(), + nullable=False) + op.alter_column('LearningResult', 'correct_responses_pattern', + existing_type=sa.VARCHAR(), + nullable=False) + op.alter_column('LearningResult', 'created_time', + existing_type=postgresql.TIMESTAMP(), type_=sa.FLOAT(), + postgresql_using="date_part('epoch',created_time)::float") + op.alter_column('LearningResult', 'duration', + existing_type=sa.VARCHAR(), + nullable=False) + op.alter_column('LearningResult', 'extensions', + existing_type=sa.VARCHAR(), + nullable=False) + op.alter_column('LearningResult', 'interaction_type', + existing_type=sa.VARCHAR(), + nullable=False) + op.alter_column('LearningResult', 'object_definition_description', + existing_type=sa.VARCHAR(), + nullable=False) + op.alter_column('LearningResult', 'object_definition_type', + existing_type=sa.VARCHAR(), + nullable=False) + op.alter_column('LearningResult', 'object_object_type', + existing_type=sa.VARCHAR(), + nullable=False) + op.alter_column('LearningResult', 'response', + existing_type=sa.VARCHAR(), + nullable=False) + op.alter_column('LearningResult', 'score_max', + existing_type=sa.INTEGER(), + nullable=False) + op.alter_column('LearningResult', 'score_min', + existing_type=sa.INTEGER(), + nullable=False) + op.alter_column('LearningResult', 'score_raw', + existing_type=sa.INTEGER(), + nullable=False) + op.alter_column('LearningResult', 'score_scaled', + existing_type=postgresql.DOUBLE_PRECISION(precision=53), + nullable=False) + op.alter_column('LearningResult', 'success', + existing_type=sa.BOOLEAN(), + nullable=False) + op.alter_column('LearningResult', 'verb_display', + existing_type=sa.VARCHAR(), + nullable=False) + op.alter_column('LearningResult', 'verb_id', + existing_type=sa.VARCHAR(), + nullable=False) + op.alter_column('UpdateInfo', 'created_time', + existing_type=postgresql.TIMESTAMP(), type_=sa.FLOAT(), + postgresql_using="date_part('epoch',created_time)::float", nullable=False) + op.alter_column('UpdateInfo', 'last_modified_time', + existing_type=postgresql.TIMESTAMP(), type_=sa.FLOAT(), + postgresql_using="date_part('epoch',last_modified_time)::float", nullable=False) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('UpdateInfo', 'last_modified_time', existing_type=sa.FLOAT(), + type_=postgresql.TIMESTAMP(), + postgresql_using="to_timestamp(last_modified_time)", nullable=True) + op.alter_column('UpdateInfo', 'created_time', + existing_type=sa.FLOAT(), postgresql_using="to_timestamp(created_time)", + type_=postgresql.TIMESTAMP(), nullable=True) + op.alter_column('LearningResult', 'verb_id', + existing_type=sa.VARCHAR(), + nullable=True) + op.alter_column('LearningResult', 'verb_display', + existing_type=sa.VARCHAR(), + nullable=True) + op.alter_column('LearningResult', 'success', + existing_type=sa.BOOLEAN(), + nullable=True) + op.alter_column('LearningResult', 'score_scaled', + existing_type=postgresql.DOUBLE_PRECISION(precision=53), + nullable=True) + op.alter_column('LearningResult', 'score_raw', + existing_type=sa.INTEGER(), + nullable=True) + op.alter_column('LearningResult', 'score_min', + existing_type=sa.INTEGER(), + nullable=True) + op.alter_column('LearningResult', 'score_max', + existing_type=sa.INTEGER(), + nullable=True) + op.alter_column('LearningResult', 'response', + existing_type=sa.VARCHAR(), + nullable=True) + op.alter_column('LearningResult', 'object_object_type', + existing_type=sa.VARCHAR(), + nullable=True) + op.alter_column('LearningResult', 'object_definition_type', + existing_type=sa.VARCHAR(), + nullable=True) + op.alter_column('LearningResult', 'object_definition_description', + existing_type=sa.VARCHAR(), + nullable=True) + op.alter_column('LearningResult', 'interaction_type', + existing_type=sa.VARCHAR(), + nullable=True) + op.alter_column('LearningResult', 'extensions', + existing_type=sa.VARCHAR(), + nullable=True) + op.alter_column('LearningResult', 'duration', + existing_type=sa.VARCHAR(), + nullable=True) + op.alter_column('LearningResult', 'created_time', + existing_type=sa.FLOAT(), postgresql_using="to_timestamp(created_time)", + type_=postgresql.TIMESTAMP()) + op.alter_column('LearningResult', 'correct_responses_pattern', + existing_type=sa.VARCHAR(), + nullable=True) + op.alter_column('LearningResult', 'completion', + existing_type=sa.BOOLEAN(), + nullable=True) + op.alter_column('LearningResult', 'choices', + existing_type=sa.VARCHAR(), + nullable=True) + op.alter_column('LearningResult', 'category_object_type', + existing_type=sa.VARCHAR(), + nullable=True) + op.alter_column('LearningResult', 'category_id', + existing_type=sa.VARCHAR(), + nullable=True) + op.alter_column('LearningResult', 'actor_object_type', + existing_type=sa.VARCHAR(), + nullable=True) + op.alter_column('LearningResult', 'actor_account_name', + existing_type=sa.VARCHAR(), + nullable=True) + op.add_column( + 'Exercise', sa.Column('uri', sa.VARCHAR(), autoincrement=False, nullable=False, + server_default=sa.text("random()"))) + op.create_unique_constraint('Exercise_uri_key', 'Exercise', ['uri']) + op.alter_column('Exercise', 'work_title', + existing_type=sa.VARCHAR(), + nullable=True, + existing_server_default=sa.text("''::character varying")) + op.alter_column('Exercise', 'work_author', + existing_type=sa.VARCHAR(), + nullable=True, + existing_server_default=sa.text("''::character varying")) + op.alter_column('Exercise', 'text_complexity', + existing_type=postgresql.DOUBLE_PRECISION(precision=53), + nullable=True, + existing_server_default=sa.text("'0'::double precision")) + op.alter_column('Exercise', 'search_values', + existing_type=sa.VARCHAR(), + nullable=True, + existing_server_default=sa.text("''::character varying")) + op.alter_column('Exercise', 'partially_correct_feedback', + existing_type=sa.VARCHAR(), + nullable=True) + op.alter_column('Exercise', 'last_access_time', + existing_type=sa.FLOAT(), postgresql_using="to_timestamp(last_access_time)", + type_=postgresql.TIMESTAMP(), nullable=True) + op.alter_column('Exercise', 'language', + existing_type=sa.VARCHAR(), + nullable=True, + existing_server_default=sa.text("'de'::character varying")) + op.alter_column('Exercise', 'instructions', + existing_type=sa.VARCHAR(), + nullable=True) + op.alter_column('Exercise', 'incorrect_feedback', + existing_type=sa.VARCHAR(), + nullable=True) + op.alter_column('Exercise', 'general_feedback', + existing_type=sa.VARCHAR(), + nullable=True) + op.alter_column('Exercise', 'exercise_type_translation', + existing_type=sa.VARCHAR(), + nullable=True) + op.alter_column('Exercise', 'correct_feedback', + existing_type=sa.VARCHAR(), + nullable=True) + op.add_column('Corpus', sa.Column('uri', sa.VARCHAR(), autoincrement=False, nullable=False, + server_default=sa.text("random()"))) + op.create_unique_constraint('Corpus_uri_key', 'Corpus', ['uri']) + # ### end Alembic commands ### diff --git a/mc_backend/mcserver/migrations/versions/c4e24d8aed64_.py b/mc_backend/mcserver/migrations/versions/c4e24d8aed64_.py index e3f5cc1e235be34dac7ea84c6d9615ca83b80141..fb001e579dcf8df22c580aa1c996f2137f63b30b 100644 --- a/mc_backend/mcserver/migrations/versions/c4e24d8aed64_.py +++ b/mc_backend/mcserver/migrations/versions/c4e24d8aed64_.py @@ -6,13 +6,11 @@ Create Date: 2018-12-05 09:40:08.357418 """ from datetime import datetime - from alembic import op import sqlalchemy as sa - -# revision identifiers, used by Alembic. from mcserver.app.models import ResourceType +# revision identifiers, used by Alembic. revision = 'c4e24d8aed64' down_revision = 'bde9f5b4e88d' branch_labels = None diff --git a/mc_backend/mcserver/models_auto.py b/mc_backend/mcserver/models_auto.py new file mode 100644 index 0000000000000000000000000000000000000000..9ef6420382b08a3ebcc4c1f6a2ac27eebd4e5599 --- /dev/null +++ b/mc_backend/mcserver/models_auto.py @@ -0,0 +1,757 @@ +"""Autogenerated SQLAlchemy models based on OpenAlchemy models.""" +# pylint: disable=no-member,super-init-not-called,unused-argument + +import typing + +import sqlalchemy +from sqlalchemy import orm + +from open_alchemy import models + + +class _CorpusDictBase(typing.TypedDict, total=True): + """TypedDict for properties that are required.""" + + source_urn: str + + +class CorpusDict(_CorpusDictBase, total=False): + """TypedDict for properties that are not required.""" + + author: str + cid: int + citation_level_1: str + citation_level_2: str + citation_level_3: str + title: str + + +class TCorpus(typing.Protocol): + """ + SQLAlchemy model protocol. + + Collection of texts. + + Attrs: + author: Author of the texts in the corpus. + cid: Unique identifier for the corpus. + citation_level_1: First level for citing the corpus. + citation_level_2: Second level for citing the corpus. + citation_level_3: Third level for citing the corpus. + source_urn: CTS base URN for referencing the corpus. + title: Corpus title. + + """ + + # SQLAlchemy properties + __table__: sqlalchemy.Table + __tablename__: str + query: orm.Query + + # Model properties + author: str + cid: int + citation_level_1: str + citation_level_2: str + citation_level_3: str + source_urn: str + title: str + + def __init__( + self, + source_urn: str, + author: str = "Anonymus", + cid: typing.Optional[int] = None, + citation_level_1: str = "default", + citation_level_2: str = "default", + citation_level_3: str = "default", + title: str = "Anonymus", + ) -> None: + """ + Construct. + + Args: + author: Author of the texts in the corpus. + cid: Unique identifier for the corpus. + citation_level_1: First level for citing the corpus. + citation_level_2: Second level for citing the corpus. + citation_level_3: Third level for citing the corpus. + source_urn: CTS base URN for referencing the corpus. + title: Corpus title. + + """ + ... + + @classmethod + def from_dict( + cls, + source_urn: str, + author: str = "Anonymus", + cid: typing.Optional[int] = None, + citation_level_1: str = "default", + citation_level_2: str = "default", + citation_level_3: str = "default", + title: str = "Anonymus", + ) -> "TCorpus": + """ + Construct from a dictionary (eg. a POST payload). + + Args: + author: Author of the texts in the corpus. + cid: Unique identifier for the corpus. + citation_level_1: First level for citing the corpus. + citation_level_2: Second level for citing the corpus. + citation_level_3: Third level for citing the corpus. + source_urn: CTS base URN for referencing the corpus. + title: Corpus title. + + Returns: + Model instance based on the dictionary. + + """ + ... + + @classmethod + def from_str(cls, value: str) -> "TCorpus": + """ + Construct from a JSON string (eg. a POST payload). + + Returns: + Model instance based on the JSON string. + + """ + ... + + def to_dict(self) -> CorpusDict: + """ + Convert to a dictionary (eg. to send back for a GET request). + + Returns: + Dictionary based on the model instance. + + """ + ... + + def to_str(self) -> str: + """ + Convert to a JSON string (eg. to send back for a GET request). + + Returns: + JSON string based on the model instance. + + """ + ... + + +Corpus: TCorpus = models.Corpus # type: ignore + + +class _ExerciseDictBase(typing.TypedDict, total=True): + """TypedDict for properties that are required.""" + + eid: str + last_access_time: float + + +class ExerciseDict(_ExerciseDictBase, total=False): + """TypedDict for properties that are not required.""" + + correct_feedback: str + general_feedback: str + incorrect_feedback: str + instructions: str + partially_correct_feedback: str + search_values: str + work_author: str + work_title: str + conll: str + exercise_type: str + exercise_type_translation: str + language: str + solutions: str + text_complexity: float + urn: str + + +class TExercise(typing.Protocol): + """ + SQLAlchemy model protocol. + + Data for creating and evaluating interactive exercises. + + Attrs: + correct_feedback: Feedback for successful completion of the exercise. + general_feedback: Feedback for finishing the exercise. + incorrect_feedback: Feedback for failing to complete the exercise + successfully. + instructions: Hints for how to complete the exercise. + partially_correct_feedback: Feedback for successfully completing + certain parts of the exercise. + search_values: Search queries that were used to build the exercise. + work_author: Name of the person who wrote the base text for the + exercise. + work_title: Title of the base text for the exercise. + conll: CONLL-formatted linguistic annotations represented as a single + string. + eid: Unique identifier (UUID) for the exercise. + exercise_type: Type of exercise, concerning interaction and layout. + exercise_type_translation: Localized expression of the exercise type. + language: ISO 639-1 Language Code for the localization of exercise + content. + last_access_time: When the exercise was last accessed (as POSIX + timestamp). + solutions: Correct solutions for the exercise. + text_complexity: Overall text complexity as measured by the software's + internal language analysis. + urn: CTS URN for the text passage from which the exercise was created. + + """ + + # SQLAlchemy properties + __table__: sqlalchemy.Table + __tablename__: str + query: orm.Query + + # Model properties + correct_feedback: str + general_feedback: str + incorrect_feedback: str + instructions: str + partially_correct_feedback: str + search_values: str + work_author: str + work_title: str + conll: str + eid: str + exercise_type: str + exercise_type_translation: str + language: str + last_access_time: float + solutions: str + text_complexity: float + urn: str + + def __init__( + self, + eid: str, + last_access_time: float, + correct_feedback: str = "", + general_feedback: str = "", + incorrect_feedback: str = "", + instructions: str = "", + partially_correct_feedback: str = "", + search_values: str = "[]", + work_author: str = "", + work_title: str = "", + conll: str = "", + exercise_type: str = "", + exercise_type_translation: str = "", + language: str = "de", + solutions: str = "[]", + text_complexity: float = 0, + urn: str = "", + ) -> None: + """ + Construct. + + Args: + correct_feedback: Feedback for successful completion of the + exercise. + general_feedback: Feedback for finishing the exercise. + incorrect_feedback: Feedback for failing to complete the exercise + successfully. + instructions: Hints for how to complete the exercise. + partially_correct_feedback: Feedback for successfully completing + certain parts of the exercise. + search_values: Search queries that were used to build the exercise. + work_author: Name of the person who wrote the base text for the + exercise. + work_title: Title of the base text for the exercise. + conll: CONLL-formatted linguistic annotations represented as a + single string. + eid: Unique identifier (UUID) for the exercise. + exercise_type: Type of exercise, concerning interaction and layout. + exercise_type_translation: Localized expression of the exercise + type. + language: ISO 639-1 Language Code for the localization of exercise + content. + last_access_time: When the exercise was last accessed (as POSIX + timestamp). + solutions: Correct solutions for the exercise. + text_complexity: Overall text complexity as measured by the + software's internal language analysis. + urn: CTS URN for the text passage from which the exercise was + created. + + """ + ... + + @classmethod + def from_dict( + cls, + eid: str, + last_access_time: float, + correct_feedback: str = "", + general_feedback: str = "", + incorrect_feedback: str = "", + instructions: str = "", + partially_correct_feedback: str = "", + search_values: str = "[]", + work_author: str = "", + work_title: str = "", + conll: str = "", + exercise_type: str = "", + exercise_type_translation: str = "", + language: str = "de", + solutions: str = "[]", + text_complexity: float = 0, + urn: str = "", + ) -> "TExercise": + """ + Construct from a dictionary (eg. a POST payload). + + Args: + correct_feedback: Feedback for successful completion of the + exercise. + general_feedback: Feedback for finishing the exercise. + incorrect_feedback: Feedback for failing to complete the exercise + successfully. + instructions: Hints for how to complete the exercise. + partially_correct_feedback: Feedback for successfully completing + certain parts of the exercise. + search_values: Search queries that were used to build the exercise. + work_author: Name of the person who wrote the base text for the + exercise. + work_title: Title of the base text for the exercise. + conll: CONLL-formatted linguistic annotations represented as a + single string. + eid: Unique identifier (UUID) for the exercise. + exercise_type: Type of exercise, concerning interaction and layout. + exercise_type_translation: Localized expression of the exercise + type. + language: ISO 639-1 Language Code for the localization of exercise + content. + last_access_time: When the exercise was last accessed (as POSIX + timestamp). + solutions: Correct solutions for the exercise. + text_complexity: Overall text complexity as measured by the + software's internal language analysis. + urn: CTS URN for the text passage from which the exercise was + created. + + Returns: + Model instance based on the dictionary. + + """ + ... + + @classmethod + def from_str(cls, value: str) -> "TExercise": + """ + Construct from a JSON string (eg. a POST payload). + + Returns: + Model instance based on the JSON string. + + """ + ... + + def to_dict(self) -> ExerciseDict: + """ + Convert to a dictionary (eg. to send back for a GET request). + + Returns: + Dictionary based on the model instance. + + """ + ... + + def to_str(self) -> str: + """ + Convert to a JSON string (eg. to send back for a GET request). + + Returns: + JSON string based on the model instance. + + """ + ... + + +Exercise: TExercise = models.Exercise # type: ignore + + +class _LearningResultDictBase(typing.TypedDict, total=True): + """TypedDict for properties that are required.""" + + completion: bool + correct_responses_pattern: str + created_time: float + object_definition_description: str + response: str + score_max: int + score_min: int + score_raw: int + success: bool + + +class LearningResultDict(_LearningResultDictBase, total=False): + """TypedDict for properties that are not required.""" + + actor_account_name: str + actor_object_type: str + category_id: str + category_object_type: str + choices: str + duration: str + extensions: str + interaction_type: str + object_definition_type: str + object_object_type: str + score_scaled: float + verb_display: str + verb_id: str + + +class TLearningResult(typing.Protocol): + """ + SQLAlchemy model protocol. + + Learner data for completed exercises. + + Attrs: + actor_account_name: H5P user ID, usually unique per device. + actor_object_type: Describes the kind of object that was recognized as + actor. + category_id: Link to the exercise type specification. + category_object_type: Describes the kind of object that was recognized + as exercise. + choices: JSON string containing a list of possible choices, each with + ID and description. + completion: Whether the exercise was fully processed or not. + correct_responses_pattern: JSON string containing a list of possible + solutions to the exercise, given as patterns of answers. + created_time: When the learner data was received (POSIX timestamp). + duration: How many seconds it took a learner to complete the exercise. + extensions: JSON string containing a mapping of keys and values + (usually the local content ID, i.e. a versioning mechanism). + interaction_type: Exercise type. + object_definition_description: Exercise content, possibly including + instructions. + object_definition_type: Type of object definition that is presented to + the user. + object_object_type: Type of object that is presented to the user. + response: Answer provided by the user, possibly as a pattern. + score_max: Maximum possible score to be achieved in this exercise. + score_min: Minimum score to be achieved in this exercise. + score_raw: Score that was actually achieved by the user in this + exercise. + score_scaled: Relative score (between 0 and 1) that was actually + achieved by the user in this exercise. + success: Whether the exercise was successfully completed or not. + verb_display: Type of action that was performed by the user. + verb_id: Link to the type of action that was performed by the user. + + """ + + # SQLAlchemy properties + __table__: sqlalchemy.Table + __tablename__: str + query: orm.Query + + # Model properties + actor_account_name: str + actor_object_type: str + category_id: str + category_object_type: str + choices: str + completion: bool + correct_responses_pattern: str + created_time: float + duration: str + extensions: str + interaction_type: str + object_definition_description: str + object_definition_type: str + object_object_type: str + response: str + score_max: int + score_min: int + score_raw: int + score_scaled: float + success: bool + verb_display: str + verb_id: str + + def __init__( + self, + completion: bool, + correct_responses_pattern: str, + created_time: float, + object_definition_description: str, + response: str, + score_max: int, + score_min: int, + score_raw: int, + success: bool, + actor_account_name: str = "", + actor_object_type: str = "", + category_id: str = "", + category_object_type: str = "", + choices: str = "[]", + duration: str = "PT0S", + extensions: str = "{}", + interaction_type: str = "", + object_definition_type: str = "", + object_object_type: str = "", + score_scaled: float = 0, + verb_display: str = "", + verb_id: str = "", + ) -> None: + """ + Construct. + + Args: + actor_account_name: H5P user ID, usually unique per device. + actor_object_type: Describes the kind of object that was recognized + as actor. + category_id: Link to the exercise type specification. + category_object_type: Describes the kind of object that was + recognized as exercise. + choices: JSON string containing a list of possible choices, each + with ID and description. + completion: Whether the exercise was fully processed or not. + correct_responses_pattern: JSON string containing a list of + possible solutions to the exercise, given as patterns of + answers. + created_time: When the learner data was received (POSIX timestamp). + duration: How many seconds it took a learner to complete the + exercise. + extensions: JSON string containing a mapping of keys and values + (usually the local content ID, i.e. a versioning mechanism). + interaction_type: Exercise type. + object_definition_description: Exercise content, possibly including + instructions. + object_definition_type: Type of object definition that is presented + to the user. + object_object_type: Type of object that is presented to the user. + response: Answer provided by the user, possibly as a pattern. + score_max: Maximum possible score to be achieved in this exercise. + score_min: Minimum score to be achieved in this exercise. + score_raw: Score that was actually achieved by the user in this + exercise. + score_scaled: Relative score (between 0 and 1) that was actually + achieved by the user in this exercise. + success: Whether the exercise was successfully completed or not. + verb_display: Type of action that was performed by the user. + verb_id: Link to the type of action that was performed by the user. + + """ + ... + + @classmethod + def from_dict( + cls, + completion: bool, + correct_responses_pattern: str, + created_time: float, + object_definition_description: str, + response: str, + score_max: int, + score_min: int, + score_raw: int, + success: bool, + actor_account_name: str = "", + actor_object_type: str = "", + category_id: str = "", + category_object_type: str = "", + choices: str = "[]", + duration: str = "PT0S", + extensions: str = "{}", + interaction_type: str = "", + object_definition_type: str = "", + object_object_type: str = "", + score_scaled: float = 0, + verb_display: str = "", + verb_id: str = "", + ) -> "TLearningResult": + """ + Construct from a dictionary (eg. a POST payload). + + Args: + actor_account_name: H5P user ID, usually unique per device. + actor_object_type: Describes the kind of object that was recognized + as actor. + category_id: Link to the exercise type specification. + category_object_type: Describes the kind of object that was + recognized as exercise. + choices: JSON string containing a list of possible choices, each + with ID and description. + completion: Whether the exercise was fully processed or not. + correct_responses_pattern: JSON string containing a list of + possible solutions to the exercise, given as patterns of + answers. + created_time: When the learner data was received (POSIX timestamp). + duration: How many seconds it took a learner to complete the + exercise. + extensions: JSON string containing a mapping of keys and values + (usually the local content ID, i.e. a versioning mechanism). + interaction_type: Exercise type. + object_definition_description: Exercise content, possibly including + instructions. + object_definition_type: Type of object definition that is presented + to the user. + object_object_type: Type of object that is presented to the user. + response: Answer provided by the user, possibly as a pattern. + score_max: Maximum possible score to be achieved in this exercise. + score_min: Minimum score to be achieved in this exercise. + score_raw: Score that was actually achieved by the user in this + exercise. + score_scaled: Relative score (between 0 and 1) that was actually + achieved by the user in this exercise. + success: Whether the exercise was successfully completed or not. + verb_display: Type of action that was performed by the user. + verb_id: Link to the type of action that was performed by the user. + + Returns: + Model instance based on the dictionary. + + """ + ... + + @classmethod + def from_str(cls, value: str) -> "TLearningResult": + """ + Construct from a JSON string (eg. a POST payload). + + Returns: + Model instance based on the JSON string. + + """ + ... + + def to_dict(self) -> LearningResultDict: + """ + Convert to a dictionary (eg. to send back for a GET request). + + Returns: + Dictionary based on the model instance. + + """ + ... + + def to_str(self) -> str: + """ + Convert to a JSON string (eg. to send back for a GET request). + + Returns: + JSON string based on the model instance. + + """ + ... + + +LearningResult: TLearningResult = models.LearningResult # type: ignore + + +class UpdateInfoDict(typing.TypedDict, total=True): + """TypedDict for properties that are required.""" + + created_time: float + last_modified_time: float + resource_type: str + + +class TUpdateInfo(typing.Protocol): + """ + SQLAlchemy model protocol. + + Timestamps for updates of various resources. + + Attrs: + created_time: When the resource was created (as POSIX timestamp). + last_modified_time: When the resource was last modified (as POSIX + timestamp). + resource_type: Name of the resource for which update timestamps are + indexed. + + """ + + # SQLAlchemy properties + __table__: sqlalchemy.Table + __tablename__: str + query: orm.Query + + # Model properties + created_time: float + last_modified_time: float + resource_type: str + + def __init__( + self, created_time: float, last_modified_time: float, resource_type: str + ) -> None: + """ + Construct. + + Args: + created_time: When the resource was created (as POSIX timestamp). + last_modified_time: When the resource was last modified (as POSIX + timestamp). + resource_type: Name of the resource for which update timestamps are + indexed. + + """ + ... + + @classmethod + def from_dict( + cls, created_time: float, last_modified_time: float, resource_type: str + ) -> "TUpdateInfo": + """ + Construct from a dictionary (eg. a POST payload). + + Args: + created_time: When the resource was created (as POSIX timestamp). + last_modified_time: When the resource was last modified (as POSIX + timestamp). + resource_type: Name of the resource for which update timestamps are + indexed. + + Returns: + Model instance based on the dictionary. + + """ + ... + + @classmethod + def from_str(cls, value: str) -> "TUpdateInfo": + """ + Construct from a JSON string (eg. a POST payload). + + Returns: + Model instance based on the JSON string. + + """ + ... + + def to_dict(self) -> UpdateInfoDict: + """ + Convert to a dictionary (eg. to send back for a GET request). + + Returns: + Dictionary based on the model instance. + + """ + ... + + def to_str(self) -> str: + """ + Convert to a JSON string (eg. to send back for a GET request). + + Returns: + JSON string based on the model instance. + + """ + ... + + +UpdateInfo: TUpdateInfo = models.UpdateInfo # type: ignore diff --git a/mc_backend/mocks.py b/mc_backend/mocks.py index b57ea97fc76379f65b9e105a89440ef9d1f92a69..ab883b0b54fa1783d5b365746cc8bd5374d16a57 100644 --- a/mc_backend/mocks.py +++ b/mc_backend/mocks.py @@ -1,18 +1,50 @@ import json +import logging from collections import OrderedDict from datetime import datetime from typing import List, Tuple, Dict +from unittest.mock import patch from conllu import TokenList +from flask import Flask +from flask.ctx import AppContext +from flask.testing import FlaskClient from gensim.models import Word2Vec from gensim.models.keyedvectors import Vocab from networkx import Graph from numpy.core.multiarray import ndarray +from sqlalchemy.exc import OperationalError -from mcserver import Config -from mcserver.app.models import Phenomenon, PartOfSpeech, Corpus, CitationLevel, Exercise, SolutionElement, \ - ExerciseData, GraphData, LinkMC, NodeMC, Language, Dependency, Case, AnnisResponse, Solution, TextPart, Citation -from mcserver.app.services import AnnotationService, CustomCorpusService +from mcserver import Config, TestingConfig +from mcserver.app import db, shutdown_session +from mcserver.app.models import Phenomenon, PartOfSpeech, CitationLevel, SolutionElement, ExerciseData, GraphData, \ + LinkMC, NodeMC, Language, Dependency, Case, AnnisResponse, Solution, TextPart, Citation, ExerciseMC, CorpusMC +from mcserver.app.services import AnnotationService, CustomCorpusService, TextService +from mcserver.models_auto import Corpus, Exercise, UpdateInfo + + +class MockFilterBy: + def __init__(self, do_raise: bool = False, ui: UpdateInfo = None): + self.do_raise: bool = do_raise + self.ui: UpdateInfo = ui + + def first(self): + if self.do_raise: + raise OperationalError("error", [], "") + else: + return self.ui + + +class MockQuery: + def __init__(self, do_raise: bool = False, ui: UpdateInfo = None): + self.do_raise: bool = do_raise + self.ui: UpdateInfo = ui + + def all(self): + return db.session.query(Corpus).all() + + def filter_by(self, **kwargs): + return MockFilterBy(self.do_raise, self.ui) class MockWV: @@ -42,6 +74,30 @@ class MockResponse: pass +class TestHelper: + def __init__(self, app: Flask): + self.app: Flask = app + self.app_context: AppContext = self.app.app_context() + self.client: FlaskClient = self.app.test_client() + + @staticmethod + def update_flask_app(class_name: str, app_factory: callable) -> None: + """Sets up and tears down the testing environment for each Test Case.""" + if len(Mocks.app_dict) and list(Mocks.app_dict.keys())[0] != class_name: + if Config.CORPUS_STORAGE_MANAGER: + Config.CORPUS_STORAGE_MANAGER.__exit__(None, None, None) + list(Mocks.app_dict.values())[0].app_context.pop() + shutdown_session() + db.drop_all() + Mocks.app_dict = {} + if not len(Mocks.app_dict): + with patch.object(TextService, "init_stop_words_latin"): + Mocks.app_dict[class_name] = TestHelper(app_factory(TestingConfig)) + Mocks.app_dict[class_name].app.logger.setLevel(logging.CRITICAL) + Mocks.app_dict[class_name].app.testing = True + db.session.commit() + + class Mocks: """This class contains mock objects for unit testing purposes.""" annotations: List[TokenList] = [TokenList( @@ -63,8 +119,6 @@ class Mocks: {"id": 6, "form": ".", "lemma": ".", "upostag": "PUNCT", "xpostag": "Punc", "feats": None, "head": 1, "deprel": "punct", "deps": None, "misc": None}], metadata=OrderedDict([("sent_id", "1"), ("urn", "urn:cts:latinLit:phi0448.phi001.perseus-lat2:1.1.1-1.1.1")]))] - aqls: List[str] = ["=".join([Phenomenon.partOfSpeech.value, '"{0}"'.format( - AnnotationService.phenomenon_map[Phenomenon.partOfSpeech][PartOfSpeech.verb.name][0])])] annis_response_dict: dict = {"graph_data_raw": {"directed": True, "multigraph": True, "graph": {}, "nodes": [ {"annis::node_name": "urn:custom:latinLit:proiel.pal-agr.lat:1.1.1/doc1#sent159692tok1", "annis::node_type": "node", "annis::type": "node", "annis::tok": "Pars", "udep::lemma": "pars", @@ -610,22 +664,25 @@ class Mocks: "value": {"sentence_id": 0, "token_id": 0, "content": None, "salt_id": "salt:/urn:custom:latinLit:proiel.pal-agr.lat:1.1.1/doc1#sent159692tok1"}}], "conll": "# newdoc id = /var/folders/30/yqnv6lz56r14dqhpw18knn2r0000gp/T/tmp7qn86au9\n# sent_id = 1\n# text = Caesar fortis est.\n1\tCaesar\tCaeso\tVERB\tC1|grn1|casA|gen1|stAN\tCase=Nom|Degree=Pos|Gender=Masc|Number=Sing\t2\tcsubj\t_\t_\n2\tfortis\tfortis\tADJ\tC1|grn1|casA|gen1|stAN\tCase=Nom|Degree=Pos|Gender=Masc|Number=Sing\t0\troot\troot\t_\n3\test\tsum\tAUX\tN3|modA|tem1|gen6|stAV\tMood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin|Voice=Act\t2\tcop\t_\tSpaceAfter=No\n4\t.\t.\tPUNCT\tPunc\t_\t2\tpunct\t_\t_\n\n# sent_id = 2\n# text = Galli moriuntur.\n1\tGalli\tGallus\tPRON\tF1|grn1|casJ|gen1|stPD\tCase=Nom|Degree=Pos|Gender=Masc|Number=Plur|PronType=Dem\t2\tnsubj:pass\t_\t_\n2\tmoriuntur\tmorior\tVERB\tL3|modJ|tem1|gen9|stAV\tMood=Ind|Number=Plur|Person=3|Tense=Pres|VerbForm=Fin|Voice=Pass\t0\troot\troot\tSpaceAfter=No\n3\t.\t.\tPUNCT\tPunc\t_\t2\tpunct\t_\tSpacesAfter=\\n\n\n"} + app_dict: Dict[str, TestHelper] = {} + aqls: List[str] = ["=".join([Phenomenon.partOfSpeech.value, '"{0}"'.format( + AnnotationService.phenomenon_map[Phenomenon.partOfSpeech][PartOfSpeech.verb.name][0])])] graph_data: GraphData = AnnotationService.map_graph_data(annis_response_dict["graph_data_raw"]) annis_response: AnnisResponse = AnnisResponse(graph_data=graph_data) corpora: List[Corpus] = [ - Corpus(title="title1", source_urn="urn1", author="author1", uri=Config.SERVER_URI_CORPORA + '/1', - citation_level_1=CitationLevel.default), - Corpus(title="title2", source_urn="urn2", author="author2", uri=Config.SERVER_URI_CORPORA + '/2', - citation_level_1=CitationLevel.default)] + CorpusMC.from_dict(title="title1", source_urn="urn1", author="author1", + citation_level_1=CitationLevel.default.value), + CorpusMC.from_dict(title="title2", source_urn="urn2", author="author2", + citation_level_1=CitationLevel.default.value)] cts_capabilities_xml: str = 'GetInventoryurn=urn:cts:latinLitTibullusCorpus TibullianumElegiaeElegiae, Aliorumque carminum libri tresTibullus, creator; Postgate, J. P. (John Percival), 1853- 1926, editor ' cts_passage_xml: str = 'GetPassageurn:cts:latinLit:phi0448.phi001.perseus-lat2:1.1.1-1.1.2urn:cts:latinLit:phi0448.phi001.perseus-lat2:1.1.1-1.1.2

Gallia est omnis divisa in partes tres, quarum unam incolunt Belgae, aliam Aquitani, tertiam qui ipsorum lingua Celtae, nostra Galli appellantur.

Hi omnes lingua, institutis, legibus inter se differunt. Gallos ab Aquitanis Garumna flumen, a Belgis Matrona et Sequana dividit.

' cts_passage_xml_1_level: str = 'GetPassageurn:cts:latinLit:phi0448.phi001.perseus-lat2:1.1-1.2urn:cts:latinLit:phi0448.phi001.perseus-lat2:1.1-1.2

Gallia est omnis divisa in partes tres, quarum unam incolunt Belgae, aliam Aquitani, tertiam qui ipsorum lingua Celtae, nostra Galli appellantur.

Gallia est omnis divisa in partes tres, quarum unam incolunt Belgae, aliam Aquitani, tertiam qui ipsorum lingua Celtae, nostra Galli appellantur.

Gallia est omnis divisa in partes tres, quarum unam incolunt Belgae, aliam Aquitani, tertiam qui ipsorum lingua Celtae, nostra Galli appellantur.

' cts_passage_xml_2_levels: str = 'GetPassageurn:cts:latinLit:phi0448.phi001.perseus-lat2:1.1-1.2urn:cts:latinLit:phi0448.phi001.perseus-lat2:1.1-1.2

Gallia est omnis divisa in partes tres, quarum unam incolunt Belgae, aliam Aquitani, tertiam qui ipsorum lingua Celtae, nostra Galli appellantur.

' cts_reff_xml: str = 'GetValidReffurn:cts:latinLit:phi0448.phi001.perseus-lat2:1.13urn:cts:latinLit:phi0448.phi001.perseus-lat2:1.1.1urn:cts:latinLit:phi0448.phi001.perseus-lat2:1.1.2urn:cts:latinLit:phi0448.phi001.perseus-lat2:1.1.3urn:cts:latinLit:phi0448.phi001.perseus-lat2:1.1.4urn:cts:latinLit:phi0448.phi001.perseus-lat2:1.1.5urn:cts:latinLit:phi0448.phi001.perseus-lat2:1.1.6urn:cts:latinLit:phi0448.phi001.perseus-lat2:1.1.7' - exercise: Exercise = Exercise( - eid="test", uri="/test", last_access_time=datetime.utcnow(), exercise_type='ddwtos', + exercise: Exercise = ExerciseMC.from_dict( + eid="test", last_access_time=datetime.utcnow().timestamp(), exercise_type='ddwtos', search_values=f'["{Phenomenon.case.name}={Case.accusative.name}", "{Phenomenon.dependency.name}={Dependency.object.name}", "{Phenomenon.lemma.name}=bellum", "{Phenomenon.dependency.name}={Dependency.root.name}"]', - language=Language.English, + language=Language.English.value, conll="# newdoc id = /var/folders/30/yqnv6lz56r14dqhpw18knn2r0000gp/T/tmp7qn86au9\n# newpar\n# sent_id = 1\n# text = Caesar fortis est.\n1\tCaesar\tCaeso\tVERB\tC1|grn1|casA|gen1|stAN\tCase=Nom|Degree=Pos|Gender=Masc|Number=Sing\t2\tcsubj\t_\t_\n2\tfortis\tfortis\tADJ\tC1|grn1|casA|gen1|stAN\tCase=Nom|Degree=Pos|Gender=Masc|Number=Sing\t0\troot\t_\t_\n3\test\tsum\tAUX\tN3|modA|tem1|gen6|stAV\tMood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin|Voice=Act\t2\tcop\t_\tSpaceAfter=No\n4\t.\t.\tPUNCT\tPunc\t_\t2\tpunct\t_\t_\n\n# sent_id = 2\n# text = Galli moriuntur.\n1\tGalli\tGallus\tPRON\tF1|grn1|casJ|gen1|stPD\tCase=Nom|Degree=Pos|Gender=Masc|Number=Plur|PronType=Dem\t2\tnsubj:pass\t_\t_\n2\tmoriuntur\tmorior\tVERB\tL3|modJ|tem1|gen9|stAV\tMood=Ind|Number=Plur|Person=3|Tense=Pres|VerbForm=Fin|Voice=Pass\t0\troot\t_\tSpaceAfter=No\n3\t.\t.\tPUNCT\tPunc\t_\t2\tpunct\t_\tSpacesAfter=\\n\n\n", solutions=json.dumps([ Solution(target=SolutionElement( @@ -699,7 +756,7 @@ class Mocks: 'Romanus', 'Solomon', 'amor'] raw_text: str = "Caesar fortis est. Galli moriuntur." static_exercises_udpipe_string: str = "1\tscribere\tscribere\n1\tcommovere\tcommovere\n1\tC\tC\n1\tgaudere\tgaudere\n1\tsignum\tsignum\n1\tvas\tvas\n1\tclarus\tclarus\n1\tcondicio\tcondicio\n1\tcom\tcum\n1\tprae\tprae\n1\tmovere\tmovere\n1\tducere\tducere\n1\tde\tde\n1\tcum\tcum\n1\tistam\tiste\n1\tnationum\tnatio\n1\tclarissimae\tclarus\n1\tmoderationem\tmoderatio\n1\tanimi\tanimus\n1\tomnium\tomnis\n1\tgentium\tgens\n1\tac\tac\n1\tvirtutem\tvirtus\n1\tprovinciae\tprovincia\n1\tCaesar\tCaesar\n1\test\tesse\n1\tsatis\tsatis\n1\tgovernment\tgovernment\n1\tsocius\tsocius\n1\tprovincia\tprovincia\n1\tpublicus\tpublicus\n1\tcivis\tcivis\n1\tatque\tatque" - subgraph_json: str = '{"directed":true,"exercise_id":"","exercise_type":"","frequency_analysis":[],"graph":{},"links":[],"multigraph":true,"nodes":[{"annis_node_name":"urn:custom:latinLit:proiel.pal-agr.lat:2.23.1/doc1#sent160481tok10","annis_node_type":"node","annis_tok":"quarum","annis_type":"node","id":"salt:/urn:custom:latinLit:proiel.pal-agr.lat:2.23.1/doc1#sent160481tok10","udep_lemma":"qui","udep_upostag":"PRON","udep_xpostag":"Pr","udep_feats":"Case=Gen|Gender=Fem|Number=Plur|PronType=Rel","solution":"","is_oov":null}],"solutions":[],"text_complexity":{},"uri":""}' + subgraph_json: str = '{"directed":true,"exercise_id":"","exercise_type":"","frequency_analysis":[],"graph":{},"links":[],"multigraph":true,"nodes":[{"annis_node_name":"urn:cts:latinLit:phi0448.phi001.perseus-lat2:1.1.1-1.1.1/doc1#sent1tok3","annis_node_type":"node","annis_tok":"Galli","annis_type":"node","id":"salt:/urn:cts:latinLit:phi0448.phi001.perseus-lat2:1.1.1-1.1.1/doc1#sent1tok3","udep_lemma":"Gallo","udep_upostag":"VERB","udep_xpostag":"L3|modQ|tem1|stAC","udep_feats":"Tense=Pres|VerbForm=Inf|Voice=Pass","solution":"","is_oov":null}],"solutions":[],"text_complexity":{},"uri":""}' test_args: List[str] = ["tests.py", "-test"] text_complexity_json_string: str = '{"n_w":52,"pos":11,"n_sent":3,"avg_w_per_sent":17.33,"avg_w_len":5.79,"n_punct":3,"n_types":48,"lex_den":0.73,"n_clause":1,"n_subclause":0,"n_abl_abs":0,"n_gerund":1,"n_inf":1,"n_part":1,"all":54.53}' text_list: List[Tuple[str, str]] = [("urn:cts:latinLit:phi0448.phi001.perseus-lat2:1.1.1", raw_text.split(".")[0]), diff --git a/mc_backend/requirements.txt b/mc_backend/requirements.txt index 835f6e08db43593f634401570c80af790d795677..83bed998a387459418081d84c584bed1c8071b22 100644 --- a/mc_backend/requirements.txt +++ b/mc_backend/requirements.txt @@ -1,6 +1,9 @@ alembic==1.4.2 aniso8601==8.0.0 +appdirs==1.4.4 +attrs==19.3.0 beautifulsoup4==4.9.0 +black==19.10b0 blinker==1.4 boto==2.49.0 boto3==1.12.49 @@ -10,7 +13,9 @@ certifi==2020.4.5.1 cffi==1.14.0 chardet==3.0.4 click==7.1.2 +clickclick==1.2.2 conllu==2.3.2 +connexion==2.7.0 coverage==5.1 cycler==0.10.0 decorator==4.4.2 @@ -27,10 +32,12 @@ graphannis==0.27.0 gunicorn==20.0.4 html5lib==1.0.1 idna==2.9 +inflection==0.4.0 isodate==0.6.0 itsdangerous==1.1.0 Jinja2==2.11.2 jmespath==0.9.5 +jsonschema==3.2.0 kiwisolver==1.2.0 LinkHeader==0.4.3 lxml==4.5.0 @@ -40,6 +47,9 @@ matplotlib==3.2.1 MyCapytain==3.0.2 networkx==2.4 numpy==1.18.3 +OpenAlchemy==1.1.0 +openapi-spec-validator==0.2.8 +pathspec==0.8.0 Pillow==7.1.2 psycopg2==2.8.5 psycopg2-binary==2.8.5 @@ -47,14 +57,17 @@ pycparser==2.20 PyLD==2.0.2 pyparsing==2.4.7 PyPDF2==1.26.0 +pyrsistent==0.16.0 python-dateutil==2.8.1 python-docx==0.8.10 python-dotenv==0.13.0 python-editor==1.0.4 python-rapidjson==0.9.1 pytz==2020.1 +PyYAML==5.3.1 rdflib==5.0.0 rdflib-jsonld==0.5.0 +regex==2020.5.7 reportlab==3.5.42 requests==2.23.0 s3transfer==0.3.3 @@ -63,6 +76,9 @@ six==1.14.0 smart-open==2.0.0 soupsieve==2.0 SQLAlchemy==1.3.16 +swagger-ui-bundle==0.0.6 +toml==0.10.0 +typed-ast==1.4.1 typing==3.7.4.1 urllib3==1.25.9 webencodings==0.5.1 diff --git a/mc_backend/tests.py b/mc_backend/tests.py index 460832583fc906eb524794bf5be90ea5ba826543..23136fbdcf353358e4dedc409d14df85ea7cea27 100644 --- a/mc_backend/tests.py +++ b/mc_backend/tests.py @@ -3,6 +3,7 @@ import copy import logging import ntpath import os +import uuid from collections import OrderedDict from threading import Thread from unittest.mock import patch, MagicMock, mock_open @@ -18,7 +19,7 @@ import unittest from multiprocessing import Process from unittest import TestLoader from datetime import datetime -from typing import Dict, List, Tuple, Type +from typing import Dict, List, Tuple, Type, Any from conllu import TokenList from flask import Flask @@ -29,23 +30,22 @@ from networkx import MultiDiGraph, Graph from requests import HTTPError from sqlalchemy.exc import OperationalError from sqlalchemy.orm import session -from werkzeug.exceptions import NotFound from werkzeug.wrappers import Response import csm import mcserver -from csm.app import create_csm_app +from csm import create_csm_app from mcserver.app import create_app, db, start_updater, full_init from mcserver.app.api.exerciseAPI import map_exercise_data_to_database -from mcserver.app.api.vectorNetworkAPI import add_edges, get_concept_network -from mcserver.app.models import Corpus, UpdateInfo, ResourceType, Exercise, FileType, ExerciseType, ExerciseData, \ +from mcserver.app.models import ResourceType, FileType, ExerciseType, ExerciseData, \ NodeMC, LinkMC, GraphData, Phenomenon, CustomCorpus, AnnisResponse, Solution, DownloadableFile, Language, \ VocabularyCorpus, SolutionElement, TextComplexityMeasure, FrequencyAnalysis, CitationLevel, FrequencyItem, \ - LearningResult, TextComplexity, Dependency, PartOfSpeech, Category, Choice, Actor, XapiStatement + TextComplexity, Dependency, PartOfSpeech, Choice, XapiStatement, ExerciseMC, CorpusMC from mcserver.app.services import AnnotationService, CorpusService, FileService, CustomCorpusService, DatabaseService, \ XMLservice, TextService from mcserver.config import TestingConfig, Config -from mocks import Mocks, MockResponse, MockW2V +from mcserver.models_auto import Corpus, Exercise, UpdateInfo, LearningResult +from mocks import Mocks, MockResponse, MockW2V, MockQuery, TestHelper class McTestCase(unittest.TestCase): @@ -92,33 +92,20 @@ class McTestCase(unittest.TestCase): self.start_time = time.time() if os.path.exists(Config.GRAPH_DATABASE_DIR): shutil.rmtree(Config.GRAPH_DATABASE_DIR) - patcher = patch.object(TextService, "init_stop_words_latin") - self.addCleanup(patcher.stop) - patcher.start() - self.app: Flask = create_app(TestingConfig) - self.app.logger.setLevel(logging.CRITICAL) - self.assertIsInstance(self.app, Flask) - self.app_context = self.app.app_context() - self.app_context.push() - self.app.testing = True - self.client: FlaskClient = self.app.test_client() - self.assertIsInstance(self.client, FlaskClient) - UpdateInfo.query.delete() - Corpus.query.delete() + self.class_name: str = str(self.__class__) + TestHelper.update_flask_app(self.class_name, create_app) def tearDown(self): """Finishes testing by removing the traces.""" - db.session.remove() - db.drop_all() - self.app_context.pop() print("{0}: {1} seconds".format(self.id(), "%.2f" % (time.time() - self.start_time))) @staticmethod def add_corpus(corpus: Corpus): """ Adds a corpus to the database. """ db.session.add(corpus) - UpdateInfo.query.delete() - ui_cts: UpdateInfo = UpdateInfo(resource_type=ResourceType.cts_data, last_modified_time=datetime.utcnow()) + db.session.query(UpdateInfo).delete() + ui_cts: UpdateInfo = UpdateInfo.from_dict(resource_type=ResourceType.cts_data.name, + last_modified_time=datetime.utcnow().timestamp(), created_time=1) db.session.add(ui_cts) db.session.commit() @@ -128,172 +115,182 @@ class McTestCase(unittest.TestCase): for f in [x for x in os.listdir(folder_path) if x != ".gitignore"]: os.remove(os.path.join(folder_path, f)) + def test_api_bad_request(self): + """Returns validation errors as JSON.""" + response: Response = Mocks.app_dict[self.class_name].client.get(Config.SERVER_URI_CORPORA) + self.assertEqual(response.status_code, 400) + def test_api_corpus_delete(self): """ Deletes a single corpus. """ - response: Response = self.client.delete(Mocks.corpora[0].uri) + db.session.query(Corpus).delete() + response: Response = Mocks.app_dict[self.class_name].client.delete( + f"{Config.SERVER_URI_CORPORA}/1") self.assertEqual(response.status_code, 404) McTestCase.add_corpus(Mocks.corpora[0]) - response = self.client.delete(Mocks.corpora[0].uri) + response = Mocks.app_dict[self.class_name].client.delete(f"{Config.SERVER_URI_CORPORA}/{Mocks.corpora[0].cid}") data_json: dict = json.loads(response.get_data()) - self.assertEqual(data_json, {"result": True}) - Corpus.query.delete() - UpdateInfo.query.delete() + self.assertEqual(data_json, True) + db.session.query(Corpus).delete() + db.session.query(UpdateInfo).delete() # dirty hack so we can reuse it in other tests session.make_transient(Mocks.corpora[0]) def test_api_corpus_get(self): """ Gets information about a single corpus. """ - response: Response = self.client.get(Mocks.corpora[0].uri) + response: Response = Mocks.app_dict[self.class_name].client.get( + f"{Config.SERVER_URI_CORPORA}/{Mocks.corpora[0].cid}") self.assertEqual(response.status_code, 404) McTestCase.add_corpus(Mocks.corpora[0]) - response: Response = self.client.get(Mocks.corpora[0].uri) + response: Response = Mocks.app_dict[self.class_name].client.get( + f"{Config.SERVER_URI_CORPORA}/{Mocks.corpora[0].cid}") data_json: dict = json.loads(response.get_data()) - actual_corpus: Corpus = Corpus(json_dict=data_json["corpus"]) - self.assertEqual(actual_corpus, Mocks.corpora[0]) - Corpus.query.delete() - UpdateInfo.query.delete() + old_dict: dict = Mocks.corpora[0].to_dict() + self.assertEqual(data_json, old_dict) + db.session.query(Corpus).delete() + db.session.query(UpdateInfo).delete() # dirty hack so we can reuse it in other tests session.make_transient(Mocks.corpora[0]) def test_api_corpus_list_get(self): """Adds multiple texts to the database and queries them all.""" - class MockFilterBy: - def __init__(self, do_raise: bool = False, ui: UpdateInfo = None): - self.do_raise: bool = do_raise - self.ui: UpdateInfo = ui - - def first(self): - if self.do_raise: - raise OperationalError("error", [], "") - else: - return self.ui - - def expect_result(self: McTestCase, mock: MagicMock, do_raise: bool, lut: str, result: object, + def expect_result(self: McTestCase, mock: MagicMock, do_raise: bool, lut: str, result: Any, lmt: datetime = datetime.utcnow()): - ui: UpdateInfo = UpdateInfo(resource_type=ResourceType.cts_data, last_modified_time=lmt) - mfb: MockFilterBy = MockFilterBy(do_raise, ui) - mock.query.filter_by.return_value = mfb - response: Response = self.client.get(TestingConfig.SERVER_URI_CORPORA, - query_string=dict(last_update_time=lut)) + ui: UpdateInfo = UpdateInfo.from_dict(resource_type=ResourceType.cts_data.name, + last_modified_time=lmt.timestamp(), created_time=1) + mock.session.query.return_value = MockQuery(do_raise, ui) + response: Response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_CORPORA, + query_string=dict(last_update_time=lut)) data_json = json.loads(response.get_data()) if data_json: - data_json = [Corpus(json_dict=x) for x in data_json["corpora"]] + result = [x.to_dict() for x in result] self.assertEqual(data_json, result) - with patch.object(mcserver.app.api.corpusListAPI, "UpdateInfo") as mock_update_info: - expect_result(self, mock_update_info, True, "0", None) - expect_result(self, mock_update_info, False, str(int(datetime.utcnow().timestamp() * 1000)), None, + with patch.object(mcserver.app.api.corpusListAPI, "db") as mock_db: + expect_result(self, mock_db, True, "0", None) + expect_result(self, mock_db, False, str(int(datetime.utcnow().timestamp() * 1000)), None, datetime.fromtimestamp(0)) db.session.add_all(Mocks.corpora) db.session.commit() - expect_result(self, mock_update_info, False, "0", Mocks.corpora, datetime.fromtimestamp(time.time())) - Corpus.query.delete() - UpdateInfo.query.delete() + expect_result(self, mock_db, False, "0", Mocks.corpora, datetime.fromtimestamp(time.time())) + db.session.query(Corpus).delete() + db.session.query(UpdateInfo).delete() # dirty hack so we can reuse it in other tests session.make_transient(Mocks.corpora[0]) - def test_api_corpus_put(self): + def test_api_corpus_patch(self): """ Changes information about a single corpus. """ - response: Response = self.client.put(Mocks.corpora[0].uri) + response: Response = Mocks.app_dict[self.class_name].client.patch( + f"{Config.SERVER_URI_CORPORA}/{Mocks.corpora[0].cid}") self.assertEqual(response.status_code, 404) McTestCase.add_corpus(Mocks.corpora[0]) old_title: str = Mocks.corpora[0].title new_title: str = "new_title" - response: Response = self.client.put(Mocks.corpora[0].uri, data=dict(title=new_title)) + response: Response = Mocks.app_dict[self.class_name].client.patch( + f"{Config.SERVER_URI_CORPORA}/{Mocks.corpora[0].cid}", data=dict(title=new_title)) data_json: dict = json.loads(response.get_data()) - new_corpus: Corpus = Corpus(json_dict=data_json["corpus"]) - self.assertEqual(new_corpus.title, Mocks.corpora[0].title) + old_dict: dict = Mocks.corpora[0].to_dict() + self.assertEqual(data_json["title"], old_dict["title"]) Mocks.corpora[0].title = old_title - Corpus.query.delete() - UpdateInfo.query.delete() + db.session.query(Corpus).delete() + db.session.query(UpdateInfo).delete() # dirty hack so we can reuse it in other tests session.make_transient(Mocks.corpora[0]) def test_api_exercise_get(self): """ Retrieves an existing exercise by its exercise ID. """ - response: Response = self.client.get(Config.SERVER_URI_EXERCISE, query_string=dict(eid="")) + response: Response = Mocks.app_dict[self.class_name].client.get(Config.SERVER_URI_EXERCISE, + query_string=dict(eid="")) self.assertEqual(response.status_code, 404) old_urn: str = Mocks.exercise.urn Mocks.exercise.urn = "" db.session.add(Mocks.exercise) db.session.commit() with patch.object(CorpusService, "get_corpus", side_effect=[AnnisResponse(), Mocks.annis_response]): - response = self.client.get(Config.SERVER_URI_EXERCISE, query_string=dict(eid=Mocks.exercise.eid)) + response = Mocks.app_dict[self.class_name].client.get(Config.SERVER_URI_EXERCISE, + query_string=dict(eid=Mocks.exercise.eid)) self.assertEqual(response.status_code, 404) Mocks.exercise.urn = old_urn db.session.commit() - response = self.client.get(Config.SERVER_URI_EXERCISE, query_string=dict(eid=Mocks.exercise.eid)) + response = Mocks.app_dict[self.class_name].client.get(Config.SERVER_URI_EXERCISE, + query_string=dict(eid=Mocks.exercise.eid)) graph_dict: dict = json.loads(response.data.decode("utf-8")) ar: AnnisResponse = AnnisResponse(json_dict=graph_dict) self.assertEqual(len(ar.nodes), 52) - Exercise.query.delete() + db.session.query(Exercise).delete() session.make_transient(Mocks.exercise) - self.app_context.push() def test_api_exercise_list_get(self): """ Retrieves a list of available exercises. """ - ui_exercises: UpdateInfo = UpdateInfo(resource_type=ResourceType.exercise_list, - last_modified_time=datetime.fromtimestamp(1)) + ui_exercises: UpdateInfo = UpdateInfo.from_dict(resource_type=ResourceType.exercise_list.name, + last_modified_time=1, created_time=1) db.session.add(ui_exercises) db.session.commit() args: dict = dict(lang="fr", last_update_time=int(time.time())) - response: Response = self.client.get(TestingConfig.SERVER_URI_EXERCISE_LIST, query_string=args) + response: Response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_EXERCISE_LIST, + query_string=args) self.assertEqual(json.loads(response.get_data()), []) args["last_update_time"] = 0 db.session.add(Mocks.exercise) db.session.commit() - response = self.client.get(TestingConfig.SERVER_URI_EXERCISE_LIST, query_string=args) - exercises: List[Exercise] = [Exercise(json_dict=x) for x in json.loads(response.get_data())] # .decode("utf-8") + response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_EXERCISE_LIST, query_string=args) + exercises: List[Exercise] = [] + for exercise_dict in json.loads(response.get_data()): + exercise_dict["search_values"] = json.dumps(exercise_dict["search_values"]) + exercises.append(ExerciseMC.from_dict(**exercise_dict)) self.assertEqual(len(exercises), 1) args = dict(lang=Language.English.value, vocabulary=VocabularyCorpus.agldt.name, frequency_upper_bound=500) - response = self.client.get(TestingConfig.SERVER_URI_EXERCISE_LIST, query_string=args) - exercises: List[dict] = json.loads(response.get_data()) # .decode("utf-8") + response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_EXERCISE_LIST, query_string=args) + exercises: List[dict] = json.loads(response.get_data()) self.assertTrue(exercises[0]["matching_degree"]) - Exercise.query.delete() + db.session.query(Exercise).delete() + db.session.query(UpdateInfo).delete() session.make_transient(Mocks.exercise) def test_api_file_get(self): """Gets an existing exercise""" - ui_file: UpdateInfo = UpdateInfo(resource_type=ResourceType.file_api_clean, - last_modified_time=datetime.fromtimestamp(1)) + ui_file: UpdateInfo = UpdateInfo.from_dict(resource_type=ResourceType.file_api_clean.name, + last_modified_time=1, created_time=1) db.session.add(ui_file) db.session.commit() # create a fake old file, to be deleted on the next GET request FileService.create_tmp_file(FileType.xml, "old") args: dict = dict(type=FileType.xml.value, id=Mocks.exercise.eid, solution_indices="[0]") - response: Response = self.client.get(TestingConfig.SERVER_URI_FILE, query_string=args) + response: Response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_FILE, + query_string=args) self.assertEqual(response.status_code, 404) file_path: str = os.path.join(Config.TMP_DIRECTORY, Mocks.exercise.eid + "." + FileType.xml.value) file_content: str = "" with open(file_path, "w+") as f: f.write(file_content) - ui_file.last_modified_time = datetime.utcnow() + ui_file.last_modified_time = datetime.utcnow().timestamp() db.session.commit() del ui_file - response = self.client.get(TestingConfig.SERVER_URI_FILE, query_string=args) + response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_FILE, query_string=args) os.remove(file_path) self.assertEqual(response.data.decode("utf-8"), file_content) # add the mapped exercise to the database db.session.add(Mocks.exercise) db.session.commit() args["type"] = FileType.pdf.value - response = self.client.get(TestingConfig.SERVER_URI_FILE, query_string=args) + response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_FILE, query_string=args) # the PDFs are not deterministically reproducible because the creation date etc. is written into them self.assertTrue(response.data.startswith(Mocks.exercise_pdf)) - Exercise.query.delete() + db.session.query(Exercise).delete() session.make_transient(Mocks.exercise) def test_api_file_post(self): """ Posts exercise data to be saved temporarily or permanently on the server, e.g. for downloading. """ learning_result: str = Mocks.xapi_json_string data_dict: dict = dict(learning_result=learning_result) - self.client.post(TestingConfig.SERVER_URI_FILE, headers=Mocks.headers_form_data, data=data_dict) - lrs: List[LearningResult] = LearningResult.query.all() + Mocks.app_dict[self.class_name].client.post(TestingConfig.SERVER_URI_FILE, headers=Mocks.headers_form_data, + data=data_dict) + lrs: List[LearningResult] = db.session.query(LearningResult).all() self.assertEqual(len(lrs), 1) data_dict: dict = dict(file_type=FileType.xml.name, urn=Mocks.urn_custom, html_content="") - response: Response = self.client.post(TestingConfig.SERVER_URI_FILE, headers=Mocks.headers_form_data, - data=data_dict) + response: Response = Mocks.app_dict[self.class_name].client.post(TestingConfig.SERVER_URI_FILE, + headers=Mocks.headers_form_data, + data=data_dict) file_name = json.loads(response.data.decode("utf-8")) self.assertTrue(file_name.endswith(".xml")) os.remove(os.path.join(Config.TMP_DIRECTORY, file_name)) @@ -303,8 +300,8 @@ class McTestCase(unittest.TestCase): """ Requests a frequency analysis for a given URN. """ with patch.object(mcserver.app.services.corpusService.requests, "get", return_value=MockResponse( json.dumps([FrequencyItem(values=[], phenomena=[], count=[]).serialize()]))): - response: Response = self.client.get(TestingConfig.SERVER_URI_FREQUENCY, - query_string=dict(urn=Mocks.urn_custom)) + response: Response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_FREQUENCY, + query_string=dict(urn=Mocks.urn_custom)) result_list: List[dict] = json.loads(response.data.decode("utf-8")) fa: FrequencyAnalysis = FrequencyAnalysis(json_list=result_list) self.assertEqual(len(fa), 1) @@ -312,52 +309,55 @@ class McTestCase(unittest.TestCase): def test_api_h5p_get(self): """ Requests a H5P JSON file for a given exercise. """ args: dict = dict(eid=Mocks.exercise.eid, lang=Language.English.value, solution_indices="[0]") - response: Response = self.client.get(TestingConfig.SERVER_URI_H5P, query_string=args) + response: Response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_H5P, query_string=args) self.assertEqual(response.status_code, 404) db.session.add(Mocks.exercise) db.session.commit() - response = self.client.get(TestingConfig.SERVER_URI_H5P, query_string=args) + response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_H5P, query_string=args) self.assertIn(Mocks.h5p_json_cloze[1:-1], response.data.decode("utf-8")) Mocks.exercise.exercise_type = ExerciseType.kwic.value db.session.commit() - response = self.client.get(TestingConfig.SERVER_URI_H5P, query_string=args) + response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_H5P, query_string=args) self.assertEqual(response.status_code, 422) Mocks.exercise.exercise_type = ExerciseType.matching.value db.session.commit() - response = self.client.get(TestingConfig.SERVER_URI_H5P, query_string=args) + response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_H5P, query_string=args) self.assertIn(Mocks.h5p_json_matching[1:-1], response.data.decode("utf-8")) Mocks.exercise.exercise_type = ExerciseType.cloze.value args["lang"] = "fr" - response = self.client.get(TestingConfig.SERVER_URI_H5P, query_string=args) + response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_H5P, query_string=args) self.assertIn(Mocks.h5p_json_cloze[1:-1], response.data.decode("utf-8")) - Exercise.query.delete() + db.session.query(Exercise).delete() session.make_transient(Mocks.exercise) @patch('mcserver.app.api.kwicAPI.requests.post', side_effect=mocked_requests_post) def test_api_kwic_post(self, mock_post: MagicMock): """ Posts an AQL query to create a KWIC visualization in SVG format. """ data_dict: dict = dict(search_values=Mocks.exercise.search_values, urn=Mocks.urn_custom) - response: Response = self.client.post(TestingConfig.SERVER_URI_KWIC, headers=Mocks.headers_form_data, - data=data_dict) + response: Response = Mocks.app_dict[self.class_name].client.post( + TestingConfig.SERVER_URI_KWIC, headers=Mocks.headers_form_data, data=data_dict) self.assertTrue(response.data.startswith(Mocks.kwic_svg)) def test_api_not_found(self): """Checks the 404 response in case of an invalid API query URL.""" - with self.assertRaises(NotFound): - self.client.get("/") + response: Response = Mocks.app_dict[self.class_name].client.get("/") + self.assertEqual(response.status_code, 404) @patch('mcserver.app.services.textComplexityService.requests.post', side_effect=mocked_requests_post) @patch('mcserver.app.services.corpusService.requests.get', side_effect=mocked_requests_get) def test_api_raw_text_get(self, mock_post_tcs: MagicMock, mock_get_cs: MagicMock): """ Retrieves the raw text for a given URN. """ TestingConfig.SIMULATE_CORPUS_NOT_FOUND = True - response: Response = self.client.get(TestingConfig.SERVER_URI_RAW_TEXT, query_string=dict(urn=Mocks.urn_custom)) + response: Response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_RAW_TEXT, + query_string=dict(urn=Mocks.urn_custom)) self.assertEqual(response.status_code, 404) TestingConfig.SIMULATE_CORPUS_NOT_FOUND = False - response = self.client.get(TestingConfig.SERVER_URI_RAW_TEXT, query_string=dict(urn=Mocks.urn_custom)) + response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_RAW_TEXT, + query_string=dict(urn=Mocks.urn_custom)) self.assertEqual(len(json.loads(response.data.decode("utf-8"))["nodes"]), 52) TestingConfig.SIMULATE_EMPTY_GRAPH = True - response = self.client.get(TestingConfig.SERVER_URI_RAW_TEXT, query_string=dict(urn=Mocks.urn_custom)) + response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_RAW_TEXT, + query_string=dict(urn=Mocks.urn_custom)) self.assertEqual(response.status_code, 404) TestingConfig.SIMULATE_EMPTY_GRAPH = False @@ -389,9 +389,10 @@ class McTestCase(unittest.TestCase): with patch.object(mcserver.app.api.staticExercisesAPI.requests, "get", side_effect=[MockResponse("{}", ok=False), MockResponse("{}", content=zip_content)]): with patch.object(AnnotationService, "get_udpipe", return_value=Mocks.static_exercises_udpipe_string): - response = self.client.get(TestingConfig.SERVER_URI_STATIC_EXERCISES) + response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_STATIC_EXERCISES) self.assertEqual(response.status_code, 503) - response: Response = self.client.get(TestingConfig.SERVER_URI_STATIC_EXERCISES) + response: Response = Mocks.app_dict[self.class_name].client.get( + TestingConfig.SERVER_URI_STATIC_EXERCISES) os.remove(TestingConfig.STATIC_EXERCISES_ZIP_FILE_PATH) self.assertGreater(len(response.data.decode("utf-8")), 1900) @@ -406,25 +407,28 @@ class McTestCase(unittest.TestCase): def test_api_text_complexity_get(self, mock_get: MagicMock, mock_post: MagicMock): """ Calculates text complexity measures for a given URN. """ args: dict = dict(urn=Mocks.urn_custom, measure=TextComplexityMeasure.all.name) - response: Response = self.client.get(TestingConfig.SERVER_URI_TEXT_COMPLEXITY, query_string=args) + response: Response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_TEXT_COMPLEXITY, + query_string=args) self.assertEqual(response.data.decode("utf-8"), Mocks.text_complexity_json_string) args["measure"] = "n_w" - response = self.client.get(TestingConfig.SERVER_URI_TEXT_COMPLEXITY, query_string=args) + response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_TEXT_COMPLEXITY, + query_string=args) self.assertEqual(json.loads(response.data.decode("utf-8"))["n_w"], 52) @patch('MyCapytain.retrievers.cts5.requests.get', side_effect=mocked_requests_get) def test_api_valid_reff_get(self, mock_get: MagicMock): # """ Retrieves possible citations for a given URN. """ args: dict = dict(urn=Mocks.urn_custom[:-14]) - response: Response = self.client.get(TestingConfig.SERVER_URI_VALID_REFF, query_string=args) + response: Response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_VALID_REFF, + query_string=args) self.assertEqual(len(json.loads(response.data.decode("utf-8"))), 3) McTestCase.clear_folder(Config.REFF_CACHE_DIRECTORY) args["urn"] = f"{Mocks.urn_custom[:-13]}4" - response = self.client.get(TestingConfig.SERVER_URI_VALID_REFF, query_string=args) + response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_VALID_REFF, query_string=args) self.assertEqual(response.status_code, 404) McTestCase.clear_folder(Config.REFF_CACHE_DIRECTORY) args["urn"] = f"{Mocks.urn_custom[:-13]}abc" - response = self.client.get(TestingConfig.SERVER_URI_VALID_REFF, query_string=args) + response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_VALID_REFF, query_string=args) self.assertEqual(response.status_code, 400) McTestCase.clear_folder(Config.REFF_CACHE_DIRECTORY) TestingConfig.SIMULATE_HTTP_ERROR = True @@ -438,7 +442,8 @@ class McTestCase(unittest.TestCase): with patch.object(mcserver.app.api.vectorNetworkAPI, "add_edges", side_effect=Mocks.mock_add_eges): with patch.object(mcserver.app.api.vectorNetworkAPI.Word2Vec, "load", return_value=MockW2V()): args: dict = dict(search_regex='ueritas', nearest_neighbor_count=150, min_count=6) - response: Response = self.client.get(TestingConfig.SERVER_URI_VECTOR_NETWORK, query_string=args) + response: Response = Mocks.app_dict[self.class_name].client.get( + TestingConfig.SERVER_URI_VECTOR_NETWORK, query_string=args) svg_string: str = json.loads(response.data.decode("utf-8")) self.assertGreater(len(svg_string), 6500) @@ -448,8 +453,9 @@ class McTestCase(unittest.TestCase): with patch("mcserver.app.api.vectorNetworkAPI.open", mock_open(read_data=mock_data)): with patch.object(mcserver.app.api.vectorNetworkAPI.Word2Vec, "load", return_value=MockW2V()): data_dict: dict = dict(search_regex='uera', nearest_neighbor_count=10) - response: Response = self.client.post(TestingConfig.SERVER_URI_VECTOR_NETWORK, - headers=Mocks.headers_form_data, data=data_dict) + response: Response = Mocks.app_dict[self.class_name].client.post( + TestingConfig.SERVER_URI_VECTOR_NETWORK, + headers=Mocks.headers_form_data, data=data_dict) self.assertEqual(len(json.loads(response.data.decode("utf-8"))), 2) @patch('mcserver.app.services.corpusService.requests.get', side_effect=mocked_requests_get) @@ -458,12 +464,13 @@ class McTestCase(unittest.TestCase): """ Calculates lexical overlap between a text (specified by URN) and a static vocabulary. """ args: dict = dict(query_urn=Mocks.urn_custom, show_oov=True, vocabulary=VocabularyCorpus.agldt.name, frequency_upper_bound=500) - response: Response = self.client.get(TestingConfig.SERVER_URI_VOCABULARY, query_string=args) + response: Response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_VOCABULARY, + query_string=args) ar: AnnisResponse = AnnisResponse(json_dict=json.loads(response.data.decode("utf-8"))) self.assertTrue(NodeMC(json_dict=ar.nodes[3]).is_oov) args["show_oov"] = False args["frequency_upper_bound"] = 6000 - response = self.client.get(TestingConfig.SERVER_URI_VOCABULARY, query_string=args) + response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_VOCABULARY, query_string=args) self.assertEqual(json.loads(response.data.decode("utf-8"))[0]["matching_degree"], 90.9090909090909) def test_app_init(self): @@ -476,12 +483,12 @@ class McTestCase(unittest.TestCase): app: Flask = csm.get_app() self.assertIsInstance(app, Flask) self.assertTrue(app.config["TESTING"]) - UpdateInfo.query.delete() + db.session.query(UpdateInfo).delete() app = mcserver.get_app() self.assertIsInstance(app, Flask) self.assertTrue(app.config["TESTING"]) - self.app_context.push() - Corpus.query.delete() + Mocks.app_dict[self.class_name].app_context.push() + db.session.query(Corpus).delete() def test_create_app(self): """Creates a new Flask application and configures it. Initializes the application and the database.""" @@ -492,11 +499,11 @@ class McTestCase(unittest.TestCase): create_app(cfg) self.assertEqual(cfg.SQLALCHEMY_DATABASE_URI, Config.DATABASE_URL_LOCAL) cfg.SQLALCHEMY_DATABASE_URI = old_uri - self.app_context.push() + Mocks.app_dict[self.class_name].app_context.push() def test_get_favicon(self): """Sends the favicon to browsers, which is used, e.g., in the tabs as a symbol for our application.""" - response: Response = self.client.get(Config.SERVER_URI_FAVICON) + response: Response = Mocks.app_dict[self.class_name].client.get(Config.SERVER_URI_FAVICON) with open(os.path.join(Config.ASSETS_DIRECTORY, Config.FAVICON_FILE_NAME), "rb") as f: content: bytes = f.read() data_received: bytes = response.get_data() @@ -504,7 +511,8 @@ class McTestCase(unittest.TestCase): def test_init_corpus_storage_manager(self): """ Initializes the corpus storage manager. """ - ui_cts: UpdateInfo = UpdateInfo(resource_type=ResourceType.cts_data, last_modified_time=datetime.utcnow()) + ui_cts: UpdateInfo = UpdateInfo.from_dict(resource_type=ResourceType.cts_data.name, + last_modified_time=datetime.utcnow().timestamp(), created_time=1) db.session.add(ui_cts) db.session.commit() csm_process: Process @@ -512,18 +520,18 @@ class McTestCase(unittest.TestCase): os.environ[Config.COVERAGE_ENVIRONMENT_VARIABLE] = Config.COVERAGE_CONFIGURATION_FILE_NAME csm_process = Process(target=csm.run_app) csm_process.start() - self.app_context.push() + Mocks.app_dict[self.class_name].app_context.push() self.assertTrue(csm_process.is_alive()) csm_process.terminate() csm_process.join() self.assertFalse(csm_process.is_alive()) - UpdateInfo.query.delete() + db.session.query(UpdateInfo).delete() @patch('mcserver.app.services.textComplexityService.requests.post', side_effect=mocked_requests_post) def test_map_exercise_data_to_database(self, mock_post: MagicMock): """Maps exercise data to the database and saves it for later access.""" - ui_exercises: UpdateInfo = UpdateInfo(resource_type=ResourceType.exercise_list, - last_modified_time=datetime.fromtimestamp(1)) + ui_exercises: UpdateInfo = UpdateInfo.from_dict(resource_type=ResourceType.exercise_list.name, + last_modified_time=1, created_time=1) db.session.add(ui_exercises) db.session.commit() exercise_expected: Exercise = Mocks.exercise @@ -541,15 +549,16 @@ class McTestCase(unittest.TestCase): exercise_expected.search_values, exercise_expected.partially_correct_feedback, exercise_expected.correct_feedback, exercise_expected.instructions, exercise_expected.exercise_type_translation, exercise_expected.exercise_type, exercise_expected.solutions, - exercise_expected.uri, exercise_expected.eid] + exercise_expected.eid] actual_values: List[str] = [ exercise.conll, exercise.general_feedback, exercise.incorrect_feedback, exercise.search_values, exercise.partially_correct_feedback, exercise.correct_feedback, exercise.instructions, - exercise.exercise_type_translation, exercise.exercise_type, exercise.solutions, exercise.uri, exercise.eid] + exercise.exercise_type_translation, exercise.exercise_type, exercise.solutions, exercise.eid] self.assertEqual(expected_values, actual_values) - exercise_from_db: Exercise = Exercise.query.one() + exercise_from_db: Exercise = db.session.query(Exercise).one() self.assertEqual(exercise, exercise_from_db) - Exercise.query.delete() + db.session.query(Exercise).delete() + db.session.query(UpdateInfo).delete() session.make_transient(Mocks.exercise) @patch('MyCapytain.retrievers.cts5.requests.get', side_effect=mocked_requests_get) @@ -560,13 +569,14 @@ class McTestCase(unittest.TestCase): ec: Corpus = CorpusService.existing_corpora[0] ec.title = "" db.session.commit() - McTestCase.add_corpus(Corpus(source_urn="123")) + McTestCase.add_corpus(CorpusMC.from_dict(source_urn="123")) cls: List[CitationLevel] = [ec.citation_level_1, ec.citation_level_2, ec.citation_level_3] CorpusService.update_corpus(ec.title, ec.source_urn, ec.author, cls, ec) self.assertFalse(ec.title) CorpusService.update_corpora() self.assertTrue(ec.title) - Corpus.query.delete() + db.session.query(Corpus).delete() + db.session.query(UpdateInfo).delete() class CsmTestCase(unittest.TestCase): @@ -593,24 +603,11 @@ class CsmTestCase(unittest.TestCase): self.start_time = time.time() if os.path.exists(Config.GRAPH_DATABASE_DIR): shutil.rmtree(Config.GRAPH_DATABASE_DIR) - patcher = patch.object(TextService, "init_stop_words_latin") - self.addCleanup(patcher.stop) - patcher.start() - self.app: Flask = create_csm_app(TestingConfig) - self.app.logger.setLevel(logging.CRITICAL) - self.app_context = self.app.app_context() - self.app_context.push() - self.app.testing = True - self.client: FlaskClient = self.app.test_client() - UpdateInfo.query.delete() - Corpus.query.delete() + self.class_name: str = str(self.__class__) + TestHelper.update_flask_app(self.class_name, create_csm_app) def tearDown(self): """Finishes testing by removing the traces.""" - Config.CORPUS_STORAGE_MANAGER.__exit__(None, None, None) - db.session.remove() - db.drop_all() - self.app_context.pop() print("{0}: {1} seconds".format(self.id(), "%.2f" % (time.time() - self.start_time))) def test_api_annis_find(self): @@ -618,8 +615,8 @@ class CsmTestCase(unittest.TestCase): disk_urn: str = AnnotationService.get_disk_urn(Mocks.urn_custom) AnnotationService.map_conll_to_graph(corpus_name=Mocks.urn_custom, conll=Mocks.annotations, cs=Config.CORPUS_STORAGE_MANAGER, file_name=disk_urn) - response: Response = self.client.get(Config.SERVER_URI_ANNIS_FIND, - query_string=dict(urn=Mocks.urn_custom, aql="tok")) + response: Response = Mocks.app_dict[self.class_name].client.get( + Config.SERVER_URI_ANNIS_FIND, query_string=dict(urn=Mocks.urn_custom, aql="tok")) matches: List[str] = json.loads(response.get_data()) self.assertEqual(len(matches), 6) solutions: List[Solution] = CorpusService.get_matches(Mocks.urn_custom, ['tok ->dep tok'], @@ -636,9 +633,11 @@ class CsmTestCase(unittest.TestCase): """Gets the raw text for a specific URN.""" ret_vals: List[AnnisResponse] = [AnnisResponse(), Mocks.annis_response] with patch.object(CorpusService, "get_corpus", side_effect=ret_vals): - response: Response = self.client.get(TestingConfig.SERVER_URI_CSM, query_string=dict(urn=Mocks.urn[:5])) + response: Response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_CSM, + query_string=dict(urn=Mocks.urn[:5])) self.assertEqual(response.status_code, 404) - response: Response = self.client.get(TestingConfig.SERVER_URI_CSM, query_string=dict(urn=Mocks.urn_custom)) + response: Response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_CSM, + query_string=dict(urn=Mocks.urn_custom)) graph_data_raw: dict = json.loads(response.get_data().decode("utf-8")) graph_data: GraphData = GraphData(json_dict=graph_data_raw) text_raw = " ".join(x.annis_tok for x in graph_data.nodes) @@ -651,13 +650,14 @@ class CsmTestCase(unittest.TestCase): @patch('mcserver.app.services.textComplexityService.requests.post', side_effect=mocked_requests_post) def test_api_exercise_post(self, mock_post_cs: MagicMock, mock_post_tcs: MagicMock): """ Creates a new exercise from scratch. """ - ui_exercises: UpdateInfo = UpdateInfo(resource_type=ResourceType.exercise_list, - last_modified_time=datetime.fromtimestamp(1)) + db.session.query(UpdateInfo).delete() + ui_exercises: UpdateInfo = UpdateInfo.from_dict(resource_type=ResourceType.exercise_list.name, + last_modified_time=1, created_time=1) db.session.add(ui_exercises) db.session.commit() client: FlaskClient = CsmTestCase.set_up_mcserver() data_dict: dict = dict(urn=Mocks.exercise.urn, type=ExerciseType.matching.value, - search_values=Mocks.exercise.search_values) + search_values=Mocks.exercise.search_values, instructions='abc') response: Response = client.post(Config.SERVER_URI_EXERCISE, headers=Mocks.headers_form_data, data=data_dict) ar: AnnisResponse = AnnisResponse(json_dict=json.loads(response.data.decode("utf-8"))) self.assertEqual(len(ar.solutions), 3) @@ -665,7 +665,8 @@ class CsmTestCase(unittest.TestCase): response: Response = client.post(Config.SERVER_URI_EXERCISE, headers=Mocks.headers_form_data, data=data_dict) self.assertEqual(response.status_code, 500) Config.CORPUS_STORAGE_MANAGER_PORT = int(Config.CORPUS_STORAGE_MANAGER_PORT[:-1]) - self.app_context.push() + Mocks.app_dict[self.class_name].app_context.push() + db.session.query(UpdateInfo).delete() def test_api_frequency_get(self): """ Requests a frequency analysis for a given URN. """ @@ -675,8 +676,8 @@ class CsmTestCase(unittest.TestCase): expected_fa.append( FrequencyItem(values=[PartOfSpeech.adjective.name], phenomena=[Phenomenon.partOfSpeech], count=1)) with patch.object(CorpusService, "get_frequency_analysis", return_value=expected_fa): - response: Response = self.client.get(TestingConfig.SERVER_URI_FREQUENCY, - query_string=dict(urn=Mocks.urn_custom)) + response: Response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_FREQUENCY, + query_string=dict(urn=Mocks.urn_custom)) result_list: List[dict] = json.loads(response.data.decode("utf-8")) fa: FrequencyAnalysis = FrequencyAnalysis(json_list=result_list) self.assertEqual(fa[0].values, expected_fa[0].values) @@ -684,64 +685,82 @@ class CsmTestCase(unittest.TestCase): def test_api_subgraph_get(self): """ Retrieves subgraph data for a given URN. """ - args: dict = dict(urn=Mocks.urn_custom, aqls=['tok="quarum"'], ctx_left="0", ctx_right="0") - response: Response = self.client.get(TestingConfig.SERVER_URI_CSM_SUBGRAPH, query_string=args) - self.assertEqual(response.data.decode("utf-8"), Mocks.subgraph_json) + args: dict = dict(urn=Mocks.urn_custom, aqls=['tok="Galli"'], ctx_left="0", ctx_right="0") + response: Response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_CSM_SUBGRAPH, + query_string=args) + self.assertEqual(response.get_data(as_text=True), Mocks.subgraph_json) def test_api_subgraph_post(self): """ Retrieves KWIC-style subgraph data for a given URN. """ - args: dict = dict(urn=Mocks.urn_custom, aqls=['tok="quarum"'], ctx_left="5", ctx_right="5") - response: Response = self.client.post(TestingConfig.SERVER_URI_CSM_SUBGRAPH, data=json.dumps(args)) + args: dict = dict(urn=Mocks.urn_custom, aqls=['tok="Galli"'], ctx_left="5", ctx_right="5") + response: Response = Mocks.app_dict[self.class_name].client.post(TestingConfig.SERVER_URI_CSM_SUBGRAPH, + data=json.dumps(args)) results_list: list = json.loads(response.data.decode("utf-8")) exercise_data_list: List[ExerciseData] = [ExerciseData(json_dict=x) for x in results_list] - self.assertEqual(len(exercise_data_list[0].graph.nodes), 10) + self.assertEqual(len(exercise_data_list[0].graph.nodes), 6) with self.assertRaises(NotImplementedError): AnnotationService.get_single_subgraph("", []) def test_api_text_complexity_get(self): """ Calculates text complexity measures for a given URN. """ args: dict = dict(urn=Mocks.urn_custom, measure=TextComplexityMeasure.all.name) - response: Response = self.client.post(TestingConfig.SERVER_URI_TEXT_COMPLEXITY, data=json.dumps(args)) + response: Response = Mocks.app_dict[self.class_name].client.post(TestingConfig.SERVER_URI_TEXT_COMPLEXITY, + data=json.dumps(args)) tc: TextComplexity = TextComplexity(json_dict=json.loads(response.data.decode("utf-8"))) - self.assertEqual(tc.pos, 8) + self.assertEqual(tc.pos, 5) args["measure"] = "n_w" - response = self.client.post(TestingConfig.SERVER_URI_TEXT_COMPLEXITY, data=json.dumps(args)) + response = Mocks.app_dict[self.class_name].client.post(TestingConfig.SERVER_URI_TEXT_COMPLEXITY, + data=json.dumps(args)) tc = TextComplexity(json_dict=json.loads(response.data.decode("utf-8"))) - self.assertEqual(tc.n_w, 14) + self.assertEqual(tc.n_w, 6) @patch('mcserver.app.services.corpusService.CorpusService.update_corpora') def test_check_corpus_list_age(self, mock_update: MagicMock): """Checks whether the list of available corpora needs to be updated.""" - ui_cts: UpdateInfo = UpdateInfo(resource_type=ResourceType.cts_data, - last_modified_time=datetime.fromtimestamp(1)) + ui_cts: UpdateInfo = UpdateInfo.from_dict(resource_type=ResourceType.cts_data.name, + last_modified_time=1, created_time=1) db.session.add(ui_cts) db.session.commit() utc_now: datetime = datetime.utcnow() - DatabaseService.check_corpus_list_age(self.app) - ui_cts: UpdateInfo = UpdateInfo.query.filter_by(resource_type=ResourceType.cts_data.name).first() - self.assertGreater(ui_cts.last_modified_time, utc_now) - UpdateInfo.query.delete() + DatabaseService.check_corpus_list_age(Mocks.app_dict[self.class_name].app) + ui_cts: UpdateInfo = db.session.query(UpdateInfo).filter_by(resource_type=ResourceType.cts_data.name).first() + self.assertGreater(ui_cts.last_modified_time, utc_now.timestamp()) + db.session.query(UpdateInfo).delete() def test_corpus_storage_manager(self): """Performs an end-to-end test for the Corpus Store Manager.""" - self.client.get(TestingConfig.SERVER_URI_CSM, query_string=dict(urn=Mocks.urn_custom)) + Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_CSM, + query_string=dict(urn=Mocks.urn_custom)) data_dict: dict = dict(title=Mocks.exercise.urn, annotations=Mocks.exercise.conll, aqls=Mocks.aqls, exercise_type=ExerciseType.cloze.name, search_phenomena=[Phenomenon.partOfSpeech.name]) - first_response: Response = self.client.post(TestingConfig.SERVER_URI_CSM, data=json.dumps(data_dict)) + first_response: Response = Mocks.app_dict[self.class_name].client.post(TestingConfig.SERVER_URI_CSM, + data=json.dumps(data_dict)) # ANNIS does not create deterministically reproducible results, so we only test for a substring self.assertIn(Mocks.graph_data_raw_part, first_response.data.decode("utf-8")) - third_response: Response = self.client.post(TestingConfig.SERVER_URI_CSM, data=data_dict) + third_response: Response = Mocks.app_dict[self.class_name].client.post(TestingConfig.SERVER_URI_CSM, + data=data_dict) # Response: Bad Request self.assertEqual(third_response.status_code, 400) + def test_find_matches(self): + """ Finds matches for a given URN and AQL and returns the corresponding node IDs. """ + matches: List[str] = CorpusService.find_matches(Mocks.urn_custom[:-6] + "3.1.1", "tok", True) + self.assertEqual(len(matches), 56) + expected_matches: List[str] = ["a", "b"] + with patch.object(mcserver.app.services.corpusService.requests, "get", + return_value=MockResponse(json.dumps(expected_matches))): + matches: List[str] = CorpusService.find_matches(Mocks.urn, "") + self.assertEqual(matches, expected_matches) + def test_full_init(self): """ Fully initializes the application, including logging.""" - self.app.config["TESTING"] = False + Mocks.app_dict[self.class_name].app.config["TESTING"] = False with patch.object(CorpusService, "init_graphannis_logging"): with patch.object(mcserver.app, "start_updater") as updater_mock: - full_init(self.app, True) + full_init(Mocks.app_dict[self.class_name].app, True) self.assertEqual(updater_mock.call_count, 1) - self.app.config["TESTING"] = True + Mocks.app_dict[self.class_name].app.config["TESTING"] = True + db.session.query(UpdateInfo).delete() def test_get_annotations_from_string(self): """ Gets annotation data from a given string, be it a CoNLL string or a corpus URN. """ @@ -772,7 +791,7 @@ class CsmTestCase(unittest.TestCase): CorpusService.get_corpus(Mocks.urn_custom, True) with patch.object(CorpusService, "get_corpus", return_value=Mocks.annis_response): fa = CorpusService.get_frequency_analysis(Mocks.urn_custom, True) - self.assertEqual(len(fa), 242) + self.assertEqual(len(fa), 191) def test_get_graph(self): """ Retrieves a graph from the cache or, if not there, builds it from scratch. """ @@ -784,16 +803,16 @@ class CsmTestCase(unittest.TestCase): def test_init_updater(self): """Initializes the corpus list updater.""" with patch.object(DatabaseService, 'check_corpus_list_age', side_effect=OperationalError("", [], "")): - ui_cts: UpdateInfo = UpdateInfo(resource_type=ResourceType.cts_data, - last_modified_time=datetime.fromtimestamp(1)) + ui_cts: UpdateInfo = UpdateInfo.from_dict(resource_type=ResourceType.cts_data.name, + last_modified_time=1, created_time=1) db.session.add(ui_cts) db.session.commit() with patch.object(CorpusService, 'update_corpora') as update_mock: - t: Thread = start_updater(self.app) + t: Thread = start_updater(Mocks.app_dict[self.class_name].app) self.assertIsInstance(t, Thread) self.assertTrue(t.is_alive()) time.sleep(0.1) - UpdateInfo.query.delete() + db.session.query(UpdateInfo).delete() assert not update_mock.called def test_process_corpus_data(self): @@ -806,6 +825,11 @@ class CsmTestCase(unittest.TestCase): search_phenomena=[Phenomenon.partOfSpeech]) gd: GraphData = AnnotationService.map_graph_data(result["graph_data_raw"]) self.assertEqual(len(gd.nodes), len(Mocks.nodes)) + urn_parts: List[str] = Mocks.urn_custom.split(":") + base_urn: str = Mocks.urn_custom.replace(":" + urn_parts[-1], "") + target_corpus: CustomCorpus = next( + (x for x in CustomCorpusService.custom_corpora if x.corpus.source_urn == base_urn), None) + CustomCorpusService.init_custom_corpus(target_corpus) text_parts_list: List[Tuple[str, str]] = CorpusService.load_text_list(Mocks.urn_custom) self.assertEqual(len(text_parts_list), 1) @@ -820,22 +844,16 @@ class CommonTestCase(unittest.TestCase): def setUp(self): """Initializes the testing environment.""" self.start_time = time.time() - if os.path.exists(Config.GRAPH_DATABASE_DIR): - shutil.rmtree(Config.GRAPH_DATABASE_DIR) - with patch.object(TextService, "init_stop_words_latin"): - self.app: Flask = create_app(TestingConfig) - self.app_context = self.app.app_context() - self.app_context.push() + self.class_name: str = str(self.__class__) + TestHelper.update_flask_app(self.class_name, create_app) def tearDown(self): """Finishes testing by removing the traces.""" - db.session.remove() - db.drop_all() - self.app_context.pop() print("{0}: {1} seconds".format(self.id(), "%.2f" % (time.time() - self.start_time))) def test_add_edges(self): """Adds edges to an existing graph based on a list of keys and constraints to their similarity and frequency.""" + from mcserver.app.api.vectorNetworkAPI import add_edges w2v: Word2Vec = Word2Vec([x.split() for x in Mocks.raw_text.split(". ")], min_count=1, sample=0) graph: Graph = Graph() add_edges(["fortis"], w2v, 2, 1, graph) @@ -857,8 +875,9 @@ class CommonTestCase(unittest.TestCase): def test_create_xml_string(self): """Exports the exercise data to the Moodle XML format. See https://docs.moodle.org/35/en/Moodle_XML_format .""" - xml_string: str = XMLservice.create_xml_string(Exercise(exercise_type=ExerciseType.matching.value), [], - FileType.pdf, []) + xml_string: str = XMLservice.create_xml_string( + ExerciseMC.from_dict(exercise_type=ExerciseType.matching.value, last_access_time=0, eid=str(uuid.uuid4())), + [], FileType.pdf, []) self.assertEqual(xml_string, Mocks.exercise_xml) def test_dependency_imports(self): @@ -883,16 +902,9 @@ class CommonTestCase(unittest.TestCase): new_text_parts = CustomCorpusService.extract_custom_corpus_text(Mocks.text_parts, ["1"], ["3"], "") self.assertEqual(new_text_parts[0][1], Mocks.text_parts[0].text_value) - def test_find_matches(self): - """ Finds matches for a given URN and AQL and returns the corresponding node IDs. """ - expected_matches: List[str] = ["a", "b"] - with patch.object(mcserver.app.services.corpusService.requests, "get", - return_value=MockResponse(json.dumps(expected_matches))): - matches: List[str] = CorpusService.find_matches(Mocks.urn, "") - self.assertEqual(matches, expected_matches) - def test_get_concept_network(self): """Extracts a network of words from vector data in an AI model.""" + from mcserver.app.api.vectorNetworkAPI import get_concept_network with patch.object(mcserver.app.api.vectorNetworkAPI, "add_edges", side_effect=Mocks.mock_add_eges): with patch.object(mcserver.app.api.vectorNetworkAPI.Word2Vec, "load", return_value=MockW2V()): svg_string: str = get_concept_network("ueritas", highlight_regex_string="uera") @@ -1010,9 +1022,10 @@ class CommonTestCase(unittest.TestCase): McTestCase.add_corpus(old_corpus) del old_corpus DatabaseService.init_db_corpus() - corpus: Corpus = Corpus.query.filter_by(source_urn=cc.corpus.source_urn).first() + corpus: Corpus = db.session.query(Corpus).filter_by(source_urn=cc.corpus.source_urn).first() self.assertEqual(corpus.title, cc.corpus.title) - Corpus.query.delete() + db.session.query(Corpus).delete() + db.session.query(UpdateInfo).delete() def test_init_stop_words_latin(self): """Initializes the stop words list for Latin texts and caches it if necessary.""" @@ -1107,7 +1120,8 @@ class CommonTestCase(unittest.TestCase): self.assertFalse(Mocks.corpora[0] == Mocks.corpora[1]) self.assertFalse(Mocks.corpora[0] == "") self.assertTrue(Mocks.exercise.__repr__().startswith(" { + .then((data: object[]) => { if (data) { - const corpusList: CorpusMC[] = data['corpora'] as CorpusMC[]; + const corpusList: CorpusMC[] = data as CorpusMC[]; this.storage.set(configMC.localStorageKeyCorpora, JSON.stringify(corpusList)).then(); this.storage.get(configMC.localStorageKeyUpdateInfo).then((jsonString: string) => { const updateInfo: UpdateInfo = JSON.parse(jsonString) as UpdateInfo; @@ -177,7 +177,7 @@ export class CorpusService { return resolve(); }); } - }, async (error: HttpErrorResponse) => { + }, async () => { this.loadCorporaFromLocalStorage().then(() => { return reject(); }); @@ -185,6 +185,14 @@ export class CorpusService { }); } + getCorpusListFromJSONstring(jsonString: string): CorpusMC[] { + let jsonObject: object = JSON.parse(jsonString); + // backwards compatibility + const corpusProp = 'corpora'; + jsonObject = jsonObject.hasOwnProperty(corpusProp) ? jsonObject[corpusProp] : jsonObject; + return jsonObject as CorpusMC[]; + } + getCTStextPassage(urn: string): Promise { return new Promise(((resolve, reject) => { const url: string = configMC.backendBaseUrl + configMC.backendApiRawtextPath; @@ -387,8 +395,7 @@ export class CorpusService { return new Promise(resolve => { this.storage.get(configMC.localStorageKeyCorpora).then((jsonString: string) => { if (jsonString) { - const corpusList: CorpusMC[] = JSON.parse(jsonString) as CorpusMC[]; - this.processCorpora(corpusList); + this.processCorpora(this.getCorpusListFromJSONstring(jsonString)); } return resolve(); }); diff --git a/mc_frontend/src/app/exercise-list/exercise-list.page.ts b/mc_frontend/src/app/exercise-list/exercise-list.page.ts index dac3d31f499552f1493ff5497efb12ea6c33db99..a26e10cedbcf199e1b338d19a510678e2180c9fd 100644 --- a/mc_frontend/src/app/exercise-list/exercise-list.page.ts +++ b/mc_frontend/src/app/exercise-list/exercise-list.page.ts @@ -103,7 +103,8 @@ export class ExerciseListPage implements OnInit { this.helperService.makeGetRequest(this.http, this.toastCtrl, url, params).then((exercises: ExerciseMC[]) => { updateInfo.exerciseList = new Date().getTime(); this.storage.set(configMC.localStorageKeyUpdateInfo, JSON.stringify(updateInfo)).then(); - state.exerciseList = this.availableExercises = this.exercises = exercises; + const newExercise: ExerciseMC[] = exercises.length ? exercises : state.exerciseList; + state.exerciseList = this.availableExercises = this.exercises = newExercise; this.helperService.saveApplicationState(state).then(); this.processExercises(); return resolve(); diff --git a/mc_frontend/src/app/models/corpusMC.ts b/mc_frontend/src/app/models/corpusMC.ts index 0762ae5e89c5c570659250692387358146be424e..b80b2a54b51ba836a106223326c346b82cdae86b 100644 --- a/mc_frontend/src/app/models/corpusMC.ts +++ b/mc_frontend/src/app/models/corpusMC.ts @@ -6,7 +6,6 @@ export class CorpusMC { public cid: number; public source_urn: string; public title: string; - public uri: string; public citation_level_1: string; public citation_level_2: string; public citation_level_3: string; diff --git a/mc_frontend/src/app/models/exerciseMC.ts b/mc_frontend/src/app/models/exerciseMC.ts index 3ac08e64b16fa349750feb29bfcfb6ea026c29cf..9c3fc433a3b7f55812332b64bc5d9c4a191f04fb 100644 --- a/mc_frontend/src/app/models/exerciseMC.ts +++ b/mc_frontend/src/app/models/exerciseMC.ts @@ -16,7 +16,6 @@ export class ExerciseMC { public search_values: string[]; public solutions: Solution[]; public text_complexity: number; - public uri: string; public work_author: string; public work_title: string; diff --git a/mc_frontend/src/app/models/mockMC.ts b/mc_frontend/src/app/models/mockMC.ts index d6f66add530d984cdda2edd760dbe17a5828d8f3..94d0c26e16dced1a21ac801384f4f36472930cb9 100644 --- a/mc_frontend/src/app/models/mockMC.ts +++ b/mc_frontend/src/app/models/mockMC.ts @@ -1,6 +1,6 @@ import {CorpusMC} from './corpusMC'; import {ExerciseMC} from './exerciseMC'; -import {MoodleExerciseType, PartOfSpeechValue, Phenomenon} from './enum'; +import {PartOfSpeechValue, Phenomenon} from './enum'; import {FrequencyItem} from './frequencyItem'; import {ApplicationState} from './applicationState'; import {TextData} from './textData'; @@ -11,16 +11,13 @@ import StatementBase from './xAPI/StatementBase'; import Result from './xAPI/Result'; import Score from './xAPI/Score'; import {TextRange} from './textRange'; -import {Citation} from './citation'; export default class MockMC { - static apiResponseCorporaGet: object = { - corpora: [new CorpusMC({ - author: 'author', - source_urn: 'urn', - title: 'title', - })] - }; + static apiResponseCorporaGet: CorpusMC[] = [new CorpusMC({ + author: 'author', + source_urn: 'urn', + title: 'title', + })]; static apiResponseFrequencyAnalysisGet: FrequencyItem[] = [new FrequencyItem({ phenomena: [Phenomenon.partOfSpeech.toString()], values: [PartOfSpeechValue.adjective.toString()] diff --git a/mc_frontend/src/app/vocabulary.service.spec.ts b/mc_frontend/src/app/vocabulary.service.spec.ts index c7e446d1cc4d6b61675c2f566be00332fc23f9cf..311b97df79ceef570b5c506104ccc346e4e3661a 100644 --- a/mc_frontend/src/app/vocabulary.service.spec.ts +++ b/mc_frontend/src/app/vocabulary.service.spec.ts @@ -34,7 +34,6 @@ describe('VocabularyService', () => { const error: HttpErrorResponse = new HttpErrorResponse({status: 500}); const requestSpy: Spy = spyOn(vocabularyService.helperService, 'makeGetRequest') .and.callFake(() => Promise.reject(error)); - // result: AnnisResponse | Sentence[] vocabularyService.getVocabularyCheck('', false).then(() => { }, async (response: HttpErrorResponse) => { expect(response.status).toBe(500); @@ -48,7 +47,6 @@ describe('VocabularyService', () => { it('should be initialized', () => { vocabularyService.ngOnInit(); expect(Object.keys(vocabularyService.refVocMap).length).toBe(4); - // vocabularyService.currentReferenceVocabulary = VocabularyCorpus.bws; expect(vocabularyService.getCurrentReferenceVocabulary().totalCount).toBe(1276); expect(vocabularyService.getPossibleSubCount()).toBe(500); }); diff --git a/mc_frontend/src/app/vocabulary.service.ts b/mc_frontend/src/app/vocabulary.service.ts index f9af782a11ed65593cd6c8c4a1e4fd8a1fbfc288..d2fb10ed4ec404e2db7185623b79476cbd7fa7a3 100644 --- a/mc_frontend/src/app/vocabulary.service.ts +++ b/mc_frontend/src/app/vocabulary.service.ts @@ -25,6 +25,7 @@ export class VocabularyService implements OnInit { constructor(public http: HttpClient, public toastCtrl: ToastController, public helperService: HelperService) { + this.ngOnInit(); } getCurrentReferenceVocabulary(): Vocabulary {