From c8309ce64572daf2b3369f0f2f17773befbeca3f Mon Sep 17 00:00:00 2001
From: Konstantin Schulz <schulzkx@hu-berlin.de>
Date: Thu, 22 Feb 2024 10:44:58 +0100
Subject: [PATCH] upgrade to postgres 16
---
docker-compose.yml | 2 +-
mc_backend/README.md | 4 -
mc_backend/app.py | 9 +-
mc_backend/mcserver/__init__.py | 4 +-
mc_backend/mcserver/app/__init__.py | 31 ++-
mc_backend/mcserver/app/api/exerciseAPI.py | 4 +-
mc_backend/mcserver/app/api/fileAPI.py | 4 +-
mc_backend/mcserver/app/api/h5pAPI.py | 4 +-
mc_backend/mcserver/app/api/kwicAPI.py | 4 +-
mc_backend/mcserver/app/api/rawTextAPI.py | 4 +-
.../mcserver/app/api/vectorNetworkAPI.py | 4 +-
mc_backend/mcserver/app/api/vocabularyAPI.py | 4 +-
mc_backend/mcserver/app/api/zenodoAPI.py | 4 +-
mc_backend/mcserver/app/models.py | 5 +-
.../app/services/annotationService.py | 11 +-
.../mcserver/app/services/corpusService.py | 26 +-
.../app/services/customCorpusService.py | 20 +-
.../mcserver/app/services/databaseService.py | 6 +
mc_backend/mcserver/config.py | 3 +-
mc_backend/mcserver/models_auto.py | 6 +-
mc_backend/mocks.py | 18 +-
.../openapi/openapi_server/models/corpus.py | 2 +
.../openapi_server/openapi/openapi.yaml | 12 +-
mc_backend/openapi_models.yaml | 11 +-
mc_backend/requirements.txt | 60 ++--
mc_backend/tests.py | 263 +++++++++---------
mc_frontend/openapi/model/corpus.ts | 2 +-
mc_frontend/package.json | 2 +-
.../author-detail/author-detail.page.spec.ts | 2 +-
.../src/app/author/author.page.spec.ts | 2 +-
mc_frontend/src/app/corpus.service.spec.ts | 16 +-
.../app/doc-software/doc-software.page.html | 2 +-
mc_frontend/src/app/models/mockMC.ts | 3 +-
.../src/app/ranking/ranking.page.spec.ts | 10 +-
mc_frontend/src/app/ranking/ranking.page.ts | 6 +-
.../app/text-range/text-range.page.spec.ts | 41 ++-
.../app/upload-text/upload-text.component.ts | 2 +-
37 files changed, 329 insertions(+), 284 deletions(-)
diff --git a/docker-compose.yml b/docker-compose.yml
index 96f7fac..35c34c7 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -2,7 +2,7 @@ version: '3.7'
services:
db:
- image: postgres:12-alpine
+ image: postgres:16-alpine
environment:
- POSTGRES_HOST_AUTH_METHOD=trust
ports:
diff --git a/mc_backend/README.md b/mc_backend/README.md
index 8e3780c..c1eb7bf 100644
--- a/mc_backend/README.md
+++ b/mc_backend/README.md
@@ -36,10 +36,6 @@ To autogenerate a new migration script:
----------------------------------------------------------------
-# Models
-To generate class structures for this project automatically:
-1. Install OpenAPI Generator (using, e.g., `brew install openapi-generator`).
-2. Run: `openapi-generator generate -i ./mcserver/mcserver_api.yaml -g python-flask -o ./openapi/ && python openapi_generator.py`.
# Testing
To check the coverage of the current tests, run
`coverage run --rcfile=.coveragerc tests.py && coverage combine && coverage report -m`.
\ No newline at end of file
diff --git a/mc_backend/app.py b/mc_backend/app.py
index 3fb9c8f..80aab64 100644
--- a/mc_backend/app.py
+++ b/mc_backend/app.py
@@ -1,8 +1,11 @@
-from flask import Flask
+from pathlib import Path
+
+import connexion
from mcserver import get_app, get_cfg
-app: Flask = get_app()
+app: connexion.App = get_app()
if __name__ == "__main__":
- app.run(host=get_cfg().HOST_IP_MCSERVER, port=get_cfg().HOST_PORT, use_reloader=False)
+ app.run(import_string=f"{Path(__file__).stem}:app", host=get_cfg().HOST_IP_MCSERVER, port=get_cfg().HOST_PORT,
+ use_reloader=True)
diff --git a/mc_backend/mcserver/__init__.py b/mc_backend/mcserver/__init__.py
index de920a6..53b89e2 100644
--- a/mc_backend/mcserver/__init__.py
+++ b/mc_backend/mcserver/__init__.py
@@ -5,12 +5,12 @@ generating language exercises for them."""
import sys
from typing import Type
-from flask import Flask
+import connexion
from mcserver.config import Config, ProductionConfig, TestingConfig, DevelopmentConfig
from mcserver.app import create_app
-def get_app() -> Flask:
+def get_app() -> connexion.App:
return create_app(get_cfg())
diff --git a/mc_backend/mcserver/app/__init__.py b/mc_backend/mcserver/app/__init__.py
index 7c2c683..7089358 100644
--- a/mc_backend/mcserver/app/__init__.py
+++ b/mc_backend/mcserver/app/__init__.py
@@ -12,8 +12,8 @@ import connexion
import open_alchemy
import prance
from connexion import FlaskApp
+from connexion.middleware import MiddlewarePosition
from flask import Flask, got_request_exception, request, Response, send_from_directory
-from flask_cors import CORS
from flask_migrate import Migrate
from flask_sqlalchemy import SQLAlchemy
from graphannis.cs import CorpusStorageManager
@@ -22,6 +22,7 @@ from sqlalchemy import create_engine
from sqlalchemy.engine import Connection, Engine
from sqlalchemy.orm import Session, sessionmaker
from sqlalchemy_utils import database_exists
+from starlette.middleware.cors import CORSMiddleware
from mcserver.config import Config
@@ -46,7 +47,7 @@ def apply_event_handlers(app: FlaskApp):
app.app.teardown_appcontext(shutdown_session)
-def create_app(cfg: Type[Config] = Config) -> Flask:
+def create_app(cfg: Type[Config] = Config) -> connexion.App:
"""Create a new Flask application and configure it. Initialize the application and the database.
Arguments:
cfg -- the desired configuration class for the application
@@ -58,12 +59,12 @@ def create_app(cfg: Type[Config] = Config) -> Flask:
Config.GRAPH_DATABASE_DIR = os.path.join(Config.GRAPH_DATABASE_DIR_BASE, guid)
Config.GRAPHANNIS_LOG_PATH = os.path.join(Config.LOGS_DIRECTORY, f"graphannis_{guid}.log")
Config.CORPUS_STORAGE_MANAGER = CorpusStorageManager(Config.GRAPH_DATABASE_DIR)
- app: Flask = init_app_common(cfg=cfg)
+ connexion_app: connexion.App = init_app_common(cfg=cfg)
from mcserver.app.services import bp as services_bp
- app.register_blueprint(services_bp)
+ connexion_app.app.register_blueprint(services_bp)
from mcserver.app.api import bp as api_bp
- app.register_blueprint(api_bp)
- return app
+ connexion_app.app.register_blueprint(api_bp)
+ return connexion_app
def create_database_handler() -> SQLAlchemy:
@@ -119,16 +120,22 @@ def get_api_specification() -> dict:
return parser.specification
-def init_app_common(cfg: Type[Config] = Config) -> Flask:
- """ Initializes common Flask parts, e.g. CORS, configuration, database, migrations and custom corpora."""
- connexion_app: FlaskApp = connexion.FlaskApp(
- __name__, port=cfg.HOST_PORT, specification_dir=Config.MC_SERVER_DIRECTORY)
+def init_app_common(cfg: Type[Config] = Config) -> connexion.App:
+ """ Initializes common Flask parts, e.g., CORS, configuration, database, migrations and custom corpora."""
+ connexion_app: connexion.FlaskApp = connexion.FlaskApp(__name__) # , specification_dir=Config.MC_SERVER_DIRECTORY
spec: dict = get_api_specification()
connexion_app.add_api(spec)
apply_event_handlers(connexion_app)
app: Flask = connexion_app.app
# allow CORS requests for all API routes
- CORS(app) # , resources=r"/*"
+ connexion_app.add_middleware(
+ CORSMiddleware,
+ position=MiddlewarePosition.BEFORE_EXCEPTION,
+ allow_origins=["*"],
+ allow_credentials=True,
+ allow_methods=["*"],
+ allow_headers=["*"],
+ )
app.config.from_object(cfg)
app.app_context().push()
init_logging(app, Config.LOG_PATH_MCSERVER)
@@ -150,7 +157,7 @@ def init_app_common(cfg: Type[Config] = Config) -> Flask:
TextService.init_stop_words_latin()
if not cfg.TESTING:
full_init(app, cfg)
- return app
+ return connexion_app
def init_logging(app: Flask, log_file_path: str):
diff --git a/mc_backend/mcserver/app/api/exerciseAPI.py b/mc_backend/mcserver/app/api/exerciseAPI.py
index 006d1b8..afb470d 100644
--- a/mc_backend/mcserver/app/api/exerciseAPI.py
+++ b/mc_backend/mcserver/app/api/exerciseAPI.py
@@ -109,10 +109,10 @@ def map_exercise_data_to_database(exercise_data: ExerciseData, exercise_type: st
return new_exercise
-def post(exercise_data: dict) -> Union[Response, ConnexionResponse]:
+def post(body: dict) -> Union[Response, ConnexionResponse]:
""" The POST method for the Exercise REST API.
It creates a new exercise from the given data and stores it in the database. """
- ef: ExerciseForm = ExerciseForm.from_dict(exercise_data)
+ ef: ExerciseForm = ExerciseForm.from_dict(body)
ef.urn = ef.urn if ef.urn else ""
exercise_type: ExerciseType = ExerciseType(ef.type)
search_values_list: List[str] = json.loads(ef.search_values)
diff --git a/mc_backend/mcserver/app/api/fileAPI.py b/mc_backend/mcserver/app/api/fileAPI.py
index 28d8722..5d5c1a3 100644
--- a/mc_backend/mcserver/app/api/fileAPI.py
+++ b/mc_backend/mcserver/app/api/fileAPI.py
@@ -61,11 +61,11 @@ def get(id: str, type: FileType, solution_indices: List[int]) -> Union[Response,
return send_from_directory(Config.TMP_DIRECTORY, existing_file.file_name, mimetype=mime_type, as_attachment=True)
-def post(file_data: dict) -> Response:
+def post(body: dict) -> Response:
""" The POST method for the File REST API.
It writes learning results or HTML content to the disk for later access. """
- ff: FileForm = FileForm.from_dict(file_data)
+ ff: FileForm = FileForm.from_dict(body)
if ff.learning_result:
lr_dict: dict = json.loads(ff.learning_result)
for exercise_id in lr_dict:
diff --git a/mc_backend/mcserver/app/api/h5pAPI.py b/mc_backend/mcserver/app/api/h5pAPI.py
index 75c5147..88b6453 100644
--- a/mc_backend/mcserver/app/api/h5pAPI.py
+++ b/mc_backend/mcserver/app/api/h5pAPI.py
@@ -126,9 +126,9 @@ def make_h5p_archive(file_name_no_ext: str, response_dict: dict, target_dir: str
shutil.rmtree(content_dir)
-def post(h5p_data: dict):
+def post(body: dict):
""" The POST method for the H5P REST API. It offers client-side H5P exercises for download as ZIP archives. """
- h5p_form: H5PForm = H5PForm.from_dict(h5p_data)
+ h5p_form: H5PForm = H5PForm.from_dict(body)
language: Language = determine_language(h5p_form.lang)
exercise: Exercise = DatabaseService.query(Exercise, filter_by=dict(eid=h5p_form.eid), first=True)
if not exercise:
diff --git a/mc_backend/mcserver/app/api/kwicAPI.py b/mc_backend/mcserver/app/api/kwicAPI.py
index f527204..427bc27 100644
--- a/mc_backend/mcserver/app/api/kwicAPI.py
+++ b/mc_backend/mcserver/app/api/kwicAPI.py
@@ -16,10 +16,10 @@ from openapi.openapi_server.models import GraphData, Solution
from openapi.openapi_server.models.kwic_form import KwicForm
-def post(kwic_data: dict) -> Response:
+def post(body: dict) -> Response:
""" The POST method for the KWIC REST API. It provides example contexts for a given phenomenon
in a given corpus. """
- kwic_form: KwicForm = KwicForm.from_dict(kwic_data)
+ kwic_form: KwicForm = KwicForm.from_dict(body)
search_values_list: List[str] = json.loads(kwic_form.search_values)
aqls: List[str] = AnnotationService.map_search_values_to_aql(search_values_list, ExerciseType.kwic)
disk_urn: str = AnnotationService.get_disk_urn(kwic_form.urn)
diff --git a/mc_backend/mcserver/app/api/rawTextAPI.py b/mc_backend/mcserver/app/api/rawTextAPI.py
index b067701..830f348 100644
--- a/mc_backend/mcserver/app/api/rawTextAPI.py
+++ b/mc_backend/mcserver/app/api/rawTextAPI.py
@@ -21,9 +21,9 @@ def get(urn: str) -> Union[Response, ConnexionResponse]:
return NetworkService.make_json_response(ar.to_dict())
-def post(raw_text_data: dict) -> Union[Response, ConnexionResponse]:
+def post(body: dict) -> Union[Response, ConnexionResponse]:
""" Provides annotations and text complexity for arbitrary Latin texts. """
- rtf: RawTextFormExtension = RawTextFormExtension.from_dict(raw_text_data)
+ rtf: RawTextFormExtension = RawTextFormExtension.from_dict(body)
text_with_extra_whitespaces: str = TextService.insert_whitespace_before_punctuation(rtf.plain_text)
annotations_conll: str = AnnotationService.get_udpipe(text_with_extra_whitespaces)
# parse CONLL and add root dependencies as separate node annotations
diff --git a/mc_backend/mcserver/app/api/vectorNetworkAPI.py b/mc_backend/mcserver/app/api/vectorNetworkAPI.py
index 4fee4a7..8180e28 100644
--- a/mc_backend/mcserver/app/api/vectorNetworkAPI.py
+++ b/mc_backend/mcserver/app/api/vectorNetworkAPI.py
@@ -76,11 +76,11 @@ def get_concept_network(search_regex_string: str, min_count: int = 1, highlight_
return svg_string
-def post(network_data: dict) -> Response:
+def post(body: dict) -> Response:
"""
The POST method for the vector network REST API. It provides sentences whose content is similar to a given word.
"""
- vnf: VectorNetworkForm = VectorNetworkForm.from_dict(network_data)
+ vnf: VectorNetworkForm = VectorNetworkForm.from_dict(body)
nearest_neighbor_count = vnf.nearest_neighbor_count if vnf.nearest_neighbor_count else 10
w2v: Word2Vec = Word2Vec.load(Config.PANEGYRICI_LATINI_MODEL_PATH)
search_regex: Pattern[str] = re.compile(vnf.search_regex)
diff --git a/mc_backend/mcserver/app/api/vocabularyAPI.py b/mc_backend/mcserver/app/api/vocabularyAPI.py
index f7f2dd3..381baf2 100644
--- a/mc_backend/mcserver/app/api/vocabularyAPI.py
+++ b/mc_backend/mcserver/app/api/vocabularyAPI.py
@@ -55,9 +55,9 @@ def get(frequency_upper_bound: int, query_urn: str, vocabulary: str) -> Union[Re
return NetworkService.make_json_response([x.to_dict() for x in sentences])
-def post(vocabulary_data: dict):
+def post(body: dict):
""" Indicates for each token of a corpus whether it is covered by a reference vocabulary. """
- vf: VocabularyForm = VocabularyForm.from_dict(vocabulary_data)
+ vf: VocabularyForm = VocabularyForm.from_dict(body)
vc: VocabularyCorpus = VocabularyCorpus[vf.vocabulary.__str__()]
vocabulary_set: Set[str] = FileService.get_vocabulary_set(vc, vf.frequency_upper_bound)
# punctuation should count as a match because we don't want to count this as part of the vocabulary
diff --git a/mc_backend/mcserver/app/api/zenodoAPI.py b/mc_backend/mcserver/app/api/zenodoAPI.py
index 0cd951c..ce2abfd 100644
--- a/mc_backend/mcserver/app/api/zenodoAPI.py
+++ b/mc_backend/mcserver/app/api/zenodoAPI.py
@@ -40,9 +40,9 @@ def get() -> Response:
return NetworkService.make_json_response([x.to_dict() for x in zenodo_records])
-def post(zenodo_data: dict) -> Response:
+def post(body: dict) -> Response:
"""The POST method for the Zenodo REST API. It provides file URIs for exercises from the Zenodo repository."""
- zenodo_form: ZenodoForm = ZenodoForm.from_dict(zenodo_data)
+ zenodo_form: ZenodoForm = ZenodoForm.from_dict(body)
sickle: Sickle = Sickle(Config.ZENODO_API_URL)
record: Record = sickle.GetRecord(metadataPrefix=ZenodoMetadataPrefix.MARC21,
identifier=f"oai:zenodo.org:{zenodo_form.record_id}")
diff --git a/mc_backend/mcserver/app/models.py b/mc_backend/mcserver/app/models.py
index bad2694..027d237 100644
--- a/mc_backend/mcserver/app/models.py
+++ b/mc_backend/mcserver/app/models.py
@@ -169,18 +169,17 @@ class CorpusMC:
@classmethod
def from_dict(cls,
+ cid: int,
source_urn: str,
author: str = "Anonymus",
- cid: typing.Optional[int] = None,
citation_level_1: str = "default",
citation_level_2: str = "default",
citation_level_3: str = "default",
title: str = "Anonymus",
) -> TCorpus:
- # ignore CID (corpus ID) because it is going to be generated automatically
return Corpus.from_dict(
source_urn=source_urn, author=author, citation_level_1=citation_level_1,
- citation_level_2=citation_level_2, citation_level_3=citation_level_3, title=title)
+ citation_level_2=citation_level_2, citation_level_3=citation_level_3, title=title, cid=cid)
class ExerciseMC:
diff --git a/mc_backend/mcserver/app/services/annotationService.py b/mc_backend/mcserver/app/services/annotationService.py
index cf06fd6..e7cf6de 100644
--- a/mc_backend/mcserver/app/services/annotationService.py
+++ b/mc_backend/mcserver/app/services/annotationService.py
@@ -82,8 +82,7 @@ class AnnotationService:
Phenomenon.LEMMA: {}}
@staticmethod
- def add_annotations_to_graph(
- conll: List[TokenList], g: GraphUpdate, doc_name: str, doc_path: str) -> None:
+ def add_annotations_to_graph(conll: List[TokenList], g: GraphUpdate, doc_name: str, doc_path: str) -> None:
""" Adds new annotations (provided in CONLL-U format) to a networkx graph. """
current_urn: str = ""
tok_before: str = ""
@@ -304,7 +303,7 @@ class AnnotationService:
return aqls
@staticmethod
- def map_token(tok, tok_id: str, g: GraphUpdate):
+ def map_token(tok: dict, tok_id: str, g: GraphUpdate):
""" Maps a token's annotations to the graphANNIS format. """
g.add_node(tok_id)
# this is an annotation node
@@ -347,13 +346,13 @@ class AnnotationService:
node_id_set: Set[str] = set([x.id for x in graph_data.nodes])
# create set for ordering link targets
target_set: Set[str] = set(map(lambda x: x.target, ordering_links))
- # look for the node of the first token in the text, i.e. the one that is not a target of any ordering link
+ # look for the node of the first token in the text, i.e., the one that is not a target of any ordering link
source_node_id: str = list(node_id_set ^ target_set)[0]
- # create a copy of the nodes so wen can change their order and then add them back to the graph
+ # create a copy of the nodes, so we can change their order and then add them back to the graph
nodes: List[NodeMC] = graph_data.nodes
# clear the previous node order
graph_data.nodes = []
- # create a lookup table so we can retrieve ordering links easily by their source value / node id
+ # create a lookup table, so we can retrieve ordering links easily by their source value / node id
ordering_link_source_to_index_dict: Dict[str, int] = {}
for i in range(len(ordering_links)):
ordering_link_source_to_index_dict[ordering_links[i].source] = i
diff --git a/mc_backend/mcserver/app/services/corpusService.py b/mc_backend/mcserver/app/services/corpusService.py
index 6f6dfff..9407fc1 100644
--- a/mc_backend/mcserver/app/services/corpusService.py
+++ b/mc_backend/mcserver/app/services/corpusService.py
@@ -18,10 +18,12 @@ from sqlalchemy.exc import OperationalError
from mcserver.app import db
from mcserver.app.models import CitationLevel, GraphData, Solution, ExerciseType, Phenomenon, AnnisResponse, CorpusMC, \
make_solution_element_from_salt_id, FrequencyItem, ResourceType, ReferenceableText
-from mcserver.app.services import AnnotationService, XMLservice, TextService, FileService, FrequencyService, \
+from mcserver.app.services import AnnotationService, XMLservice, FileService, FrequencyService, \
CustomCorpusService, DatabaseService
from mcserver.config import Config
-from mcserver.models_auto import Corpus, UpdateInfo
+from mcserver.models_auto import Corpus, UpdateInfo, TCorpus
+from sqlalchemy import select
+from sqlalchemy.sql import functions
class CorpusService:
@@ -43,15 +45,15 @@ class CorpusService:
def add_corpus(title_value: str, urn: str, group_name_value: str,
citation_levels: List[Union[CitationLevel, str]]) -> None:
"""Adds a new corpus to the database."""
- new_corpus = CorpusMC.from_dict(title=title_value, source_urn=urn, author=group_name_value,
- citation_level_1=citation_levels[0])
+ # set the max_cid to 0 if there are no corpora in the table
+ max_cid: int = db.session.execute(select(functions.max(Corpus.cid))).first()[0] or 0
+ new_corpus: TCorpus = CorpusMC.from_dict(title=title_value, source_urn=urn, author=group_name_value,
+ citation_level_1=citation_levels[0], cid=max_cid + 1)
CorpusService.add_citation_levels(new_corpus, citation_levels)
- db.session.add(new_corpus)
- # need to commit once so the Corpus ID (cid) gets generated by the database
- db.session.commit()
# now we can build the URI from the Corpus ID
new_corpus.uri = "/{0}".format(new_corpus.cid)
- db.session.commit()
+ db.session.add(new_corpus)
+ DatabaseService.commit()
@staticmethod
def check_corpus_list_age(app: Flask) -> None:
@@ -323,7 +325,7 @@ class CorpusService:
def load_text_list(cts_urn_raw: str) -> List[ReferenceableText]:
""" Loads the text list for a new corpus. """
if CustomCorpusService.is_custom_corpus_urn(cts_urn_raw):
- # this is a custom corpus, e.g. the VIVA textbook
+ # this is a custom corpus, e.g., the VIVA textbook
return CustomCorpusService.get_custom_corpus_text(cts_urn_raw)
else:
resolver: HttpCtsRetriever = HttpCtsRetriever(Config.CTS_API_BASE_URL)
@@ -398,9 +400,9 @@ class CorpusService:
if urn not in urn_set_new:
corpus_to_delete: Corpus = DatabaseService.query(Corpus, dict(source_urn=urn), True)
db.session.delete(corpus_to_delete)
- db.session.commit()
+ DatabaseService.commit()
CorpusService.existing_corpora = DatabaseService.query(Corpus)
- db.session.commit()
+ DatabaseService.commit()
@staticmethod
def update_corpus(title_value: str, urn: str, author: str,
@@ -416,4 +418,4 @@ class CorpusService:
corpus_to_update.citation_level_1 = citation_levels[0]
corpus_to_update.source_urn = urn
CorpusService.add_citation_levels(corpus_to_update, citation_levels)
- db.session.commit()
+ DatabaseService.commit()
diff --git a/mc_backend/mcserver/app/services/customCorpusService.py b/mc_backend/mcserver/app/services/customCorpusService.py
index 15b5dde..6a9b801 100644
--- a/mc_backend/mcserver/app/services/customCorpusService.py
+++ b/mc_backend/mcserver/app/services/customCorpusService.py
@@ -17,51 +17,51 @@ class CustomCorpusService:
custom_corpora: List[CustomCorpus] = [
CustomCorpus(corpus=CorpusMC.from_dict(
title="Commentarii de bello Gallico", source_urn=Config.CUSTOM_CORPUS_PROIEL_URN_TEMPLATE.format(
- "caes-gal"), author="C. Iulius Caesar", citation_level_1=CitationLevel.book.value,
+ "caes-gal"), author="C. Iulius Caesar", cid=-1, citation_level_1=CitationLevel.book.value,
citation_level_2=CitationLevel.chapter.value, citation_level_3=CitationLevel.section.value),
file_path=Config.CUSTOM_CORPUS_CAES_GAL_FILE_PATH),
CustomCorpus(corpus=CorpusMC.from_dict(
title="Epistulae ad Atticum", source_urn=Config.CUSTOM_CORPUS_PROIEL_URN_TEMPLATE.format(
- "cic-att"), author="M. Tullius Cicero", citation_level_1=CitationLevel.book.value,
+ "cic-att"), author="M. Tullius Cicero", cid=-2, citation_level_1=CitationLevel.book.value,
citation_level_2=CitationLevel.letter.value, citation_level_3=CitationLevel.section.value),
file_path=Config.CUSTOM_CORPUS_CIC_ATT_FILE_PATH),
CustomCorpus(corpus=CorpusMC.from_dict(
title="De officiis", source_urn=Config.CUSTOM_CORPUS_PROIEL_URN_TEMPLATE.format(
- "cic-off"), author="M. Tullius Cicero", citation_level_1=CitationLevel.book.value,
+ "cic-off"), author="M. Tullius Cicero", cid=-3, citation_level_1=CitationLevel.book.value,
citation_level_2=CitationLevel.section.value, citation_level_3=CitationLevel.default.value),
file_path=Config.CUSTOM_CORPUS_CIC_OFF_FILE_PATH),
CustomCorpus(corpus=CorpusMC.from_dict(
title="Vulgata (Novum Testamentum)", source_urn=Config.CUSTOM_CORPUS_PROIEL_URN_TEMPLATE.format(
- "latin-nt"), author="Hieronymus", citation_level_1=CitationLevel.chapter.value,
+ "latin-nt"), author="Hieronymus", cid=-4, citation_level_1=CitationLevel.chapter.value,
citation_level_2=CitationLevel.section.value, citation_level_3=CitationLevel.default.value),
file_path=Config.CUSTOM_CORPUS_LATIN_NT_FILE_PATH),
CustomCorpus(corpus=CorpusMC.from_dict(
title="Opus Agriculturae", source_urn=Config.CUSTOM_CORPUS_PROIEL_URN_TEMPLATE.format(
- "pal-agr"), author="Palladius", citation_level_1=CitationLevel.book.value,
+ "pal-agr"), author="Palladius", cid=-5, citation_level_1=CitationLevel.book.value,
citation_level_2=CitationLevel.chapter.value, citation_level_3=CitationLevel.section.value),
file_path=Config.CUSTOM_CORPUS_PAL_AGR_FILE_PATH),
CustomCorpus(corpus=CorpusMC.from_dict(
title="Peregrinatio Aetheriae", source_urn=Config.CUSTOM_CORPUS_PROIEL_URN_TEMPLATE.format(
- "per-aeth"), author="Peregrinatio Aetheriae", citation_level_1=CitationLevel.chapter.value,
+ "per-aeth"), author="Peregrinatio Aetheriae", cid=-6, citation_level_1=CitationLevel.chapter.value,
citation_level_2=CitationLevel.section.value, citation_level_3=CitationLevel.default.value),
file_path=Config.CUSTOM_CORPUS_PER_AET_FILE_PATH),
CustomCorpus(init_fn=InitializationService.init_custom_corpus_viva, corpus=CorpusMC.from_dict(
- title="VIVA", source_urn=Config.CUSTOM_CORPUS_VIVA_URN, author="VIVA (textbook)",
+ title="VIVA", source_urn=Config.CUSTOM_CORPUS_VIVA_URN, author="VIVA (textbook)", cid=-7,
citation_level_1=CitationLevel.book.value, citation_level_2=CitationLevel.unit.value,
citation_level_3=CitationLevel.default.value), file_path=Config.CUSTOM_CORPUS_VIVA_FILE_PATH),
CustomCorpus(init_fn=InitializationService.init_custom_corpus_pro_marcello, corpus=CorpusMC.from_dict(
- title="Pro M. Marcello", source_urn=Config.CUSTOM_CORPUS_CIC_MARC_URN, author="M. Tullius Cicero",
+ title="Pro M. Marcello", source_urn=Config.CUSTOM_CORPUS_CIC_MARC_URN, author="M. Tullius Cicero", cid=-8,
citation_level_1=CitationLevel.section.value, citation_level_2=CitationLevel.default.value,
citation_level_3=CitationLevel.default.value), file_path=Config.CUSTOM_CORPUS_CIC_MARC_FILE_PATH),
CustomCorpus(init_fn=InitializationService.init_custom_corpus_de_revolutionibus, corpus=CorpusMC.from_dict(
title="De Revolutionibus Orbium Coelestium",
- source_urn=Config.CUSTOM_CORPUS_COPERNICUS_DE_REVOLUTIONIBUS_URN, author="Nicolaus Copernicus",
+ source_urn=Config.CUSTOM_CORPUS_COPERNICUS_DE_REVOLUTIONIBUS_URN, author="Nicolaus Copernicus", cid=-9,
citation_level_1=CitationLevel.book.value, citation_level_2=CitationLevel.chapter.value,
citation_level_3=CitationLevel.default.value),
file_path=Config.CUSTOM_CORPUS_COPERNICUS_DE_REVOLUTIONIBUS_FILE_PATH),
CustomCorpus(init_fn=InitializationService.init_custom_corpus_commentariolus, corpus=CorpusMC.from_dict(
title="Commentariolus", source_urn=Config.CUSTOM_CORPUS_COPERNICUS_COMMENTARIOLUS_URN,
- author="Nicolaus Copernicus", citation_level_1=CitationLevel.chapter.value,
+ author="Nicolaus Copernicus", cid=-10, citation_level_1=CitationLevel.chapter.value,
citation_level_2=CitationLevel.default.value, citation_level_3=CitationLevel.default.value),
file_path=Config.CUSTOM_CORPUS_COPERNICUS_COMMENTARIOLUS_FILE_PATH),
]
diff --git a/mc_backend/mcserver/app/services/databaseService.py b/mc_backend/mcserver/app/services/databaseService.py
index 2cd4a6c..2deb89c 100644
--- a/mc_backend/mcserver/app/services/databaseService.py
+++ b/mc_backend/mcserver/app/services/databaseService.py
@@ -22,6 +22,12 @@ class DatabaseService:
db.session.rollback()
raise
+ @staticmethod
+ def delete(table: Union[Corpus, Exercise, LearningResult, UpdateInfo]):
+ """Deletes all rows from a table in the database."""
+ db.session.query(table).delete()
+ DatabaseService.commit()
+
@staticmethod
def has_table(table: str) -> bool:
"""Checks if a table is present in the database or not."""
diff --git a/mc_backend/mcserver/config.py b/mc_backend/mcserver/config.py
index 2a53a0d..ec8de52 100644
--- a/mc_backend/mcserver/config.py
+++ b/mc_backend/mcserver/config.py
@@ -205,7 +205,8 @@ class TestingConfig(Config):
CTS_API_GET_PASSAGE_URL = "https://cts.perseids.org/api/cts/?request=GetPassage&urn=urn:cts:latinLit:phi1351.phi002.perseus-lat1:2.2"
HOST_IP_MCSERVER = "0.0.0.0"
PRESERVE_CONTEXT_ON_EXCEPTION = False
- SERVER_NAME = Config.HOST_IP_MCSERVER + ":{0}".format(Config.HOST_PORT)
+ # reuse the server name from Starlette's TestClient implementation
+ SERVER_NAME = "testserver"
SESSION_COOKIE_DOMAIN = False
SIMULATE_HTTP_ERROR = False
SQLALCHEMY_DATABASE_URI = Config.DATABASE_URL_SQLITE
diff --git a/mc_backend/mcserver/models_auto.py b/mc_backend/mcserver/models_auto.py
index de9a6a2..2a4e37c 100644
--- a/mc_backend/mcserver/models_auto.py
+++ b/mc_backend/mcserver/models_auto.py
@@ -14,6 +14,7 @@ Base = models.Base # type: ignore
class _CorpusDictBase(typing.TypedDict, total=True):
"""TypedDict for properties that are required."""
+ cid: int
source_urn: str
@@ -21,7 +22,6 @@ class CorpusDict(_CorpusDictBase, total=False):
"""TypedDict for properties that are not required."""
author: str
- cid: int
citation_level_1: str
citation_level_2: str
citation_level_3: str
@@ -59,7 +59,7 @@ class TCorpus(typing.Protocol):
source_urn: 'sqlalchemy.Column[str]'
title: 'sqlalchemy.Column[str]'
- def __init__(self, source_urn: str, author: str = "Anonymus", cid: typing.Optional[int] = None, citation_level_1: str = "default", citation_level_2: str = "default", citation_level_3: str = "default", title: str = "Anonymus") -> None:
+ def __init__(self, cid: int, source_urn: str, author: str = "Anonymus", citation_level_1: str = "default", citation_level_2: str = "default", citation_level_3: str = "default", title: str = "Anonymus") -> None:
"""
Construct.
@@ -76,7 +76,7 @@ class TCorpus(typing.Protocol):
...
@classmethod
- def from_dict(cls, source_urn: str, author: str = "Anonymus", cid: typing.Optional[int] = None, citation_level_1: str = "default", citation_level_2: str = "default", citation_level_3: str = "default", title: str = "Anonymus") -> "TCorpus":
+ def from_dict(cls, cid: int, source_urn: str, author: str = "Anonymus", citation_level_1: str = "default", citation_level_2: str = "default", citation_level_3: str = "default", title: str = "Anonymus") -> "TCorpus":
"""
Construct from a dictionary (eg. a POST payload).
diff --git a/mc_backend/mocks.py b/mc_backend/mocks.py
index 2853c78..bfb6135 100644
--- a/mc_backend/mocks.py
+++ b/mc_backend/mocks.py
@@ -1,3 +1,4 @@
+import asyncio
import json
import logging
from collections import OrderedDict
@@ -5,6 +6,7 @@ from datetime import datetime
from typing import List, Tuple, Dict
from unittest.mock import patch
+import connexion
from conllu import TokenList
from flask import Flask
from flask.ctx import AppContext
@@ -14,6 +16,7 @@ from gensim.models.keyedvectors import Vocab
from networkx import Graph
from numpy.core.multiarray import ndarray
from requests import HTTPError
+from starlette.testclient import TestClient
from mcserver import Config, TestingConfig
from mcserver.app import db, shutdown_session
@@ -74,12 +77,13 @@ class MockW2V:
class TestHelper:
- def __init__(self, app: Flask):
- self.app: Flask = app
- self.app_context: AppContext = self.app.app_context()
+ def __init__(self, app: connexion.App):
+ self.app: connexion.App = app
+ self.app_context: AppContext = self.app.app.app_context()
self.app_context.push()
- self.client: FlaskClient = self.app.test_client()
- self.app.logger.setLevel(logging.WARNING)
+ # force Starlette/anyio to use the asyncio event loop that was patched by PyCharm
+ self.client: TestClient = self.app.test_client(backend_options={'loop_factory': asyncio.new_event_loop})
+ self.app.app.logger.setLevel(logging.WARNING)
self.app.testing = True
@staticmethod
@@ -695,9 +699,9 @@ class Mocks:
annis_response: AnnisResponse = AnnisResponse(graph_data=graph_data)
corpora: List[Corpus] = [
CorpusMC.from_dict(title="title1", source_urn="urn1", author="author1",
- citation_level_1=CitationLevel.default.value),
+ citation_level_1=CitationLevel.default.value, cid=1),
CorpusMC.from_dict(title="title2", source_urn="urn2", author="author2",
- citation_level_1=CitationLevel.default.value)]
+ citation_level_1=CitationLevel.default.value, cid=2)]
cts_capabilities_xml: str = '<GetCapabilities xmlns="http://chs.harvard.edu/xmlns/cts"><request><requestName>GetInventory</requestName><requestFilters>urn=urn:cts:latinLit</requestFilters></request><reply><ti:TextInventory xmlns:ti=\'http://chs.harvard.edu/xmlns/cts\'><ti:textgroup urn=\'urn:cts:latinLit:phi0660\' xmlns:ti=\'http://chs.harvard.edu/xmlns/cts\'><ti:groupname xml:lang=\'eng\'>Tibullus</ti:groupname><ti:groupname xml:lang=\'lat\'>Corpus Tibullianum</ti:groupname><ti:work xml:lang="lat" urn=\'urn:cts:latinLit:phi0660.phi001\' groupUrn=\'urn:cts:latinLit:phi0660\' xmlns:ti=\'http://chs.harvard.edu/xmlns/cts\'><ti:title xml:lang=\'lat\'>Elegiae</ti:title><ti:edition urn=\'urn:cts:latinLit:phi0660.phi001.perseus-lat2\' workUrn=\'urn:cts:latinLit:phi0660.phi001\' xmlns:ti=\'http://chs.harvard.edu/xmlns/cts\'><ti:label xml:lang=\'eng\'>Elegiae, Aliorumque carminum libri tres</ti:label><ti:description xml:lang=\'eng\'>Tibullus, creator; Postgate, J. P. (John Percival), 1853- 1926, editor </ti:description><ti:online><ti:citationMapping><ti:citation label="book" xpath="/tei:div[@n=\'?\']" scope="/tei:TEI/tei:text/tei:body/tei:div"><ti:citation label="poem" xpath="/tei:div[@n=\'?\']" scope="/tei:TEI/tei:text/tei:body/tei:div/tei:div[@n=\'?\']"><ti:citation label="line" xpath="//tei:l[@n=\'?\']" scope="/tei:TEI/tei:text/tei:body/tei:div/tei:div[@n=\'?\']/tei:div[@n=\'?\']"></ti:citation></ti:citation></ti:citation></ti:citationMapping></ti:online></ti:edition></ti:work><ti:work xml:lang="lat" urn=\'urn:cts:latinLit:phi0660.phi003\' groupUrn=\'urn:cts:latinLit:phi0660\' xmlns:ti=\'http://chs.harvard.edu/xmlns/cts\'> </ti:work></ti:textgroup></ti:TextInventory></reply></GetCapabilities>'
cts_passage_xml: str = '<GetPassage xmlns:tei="http://www.tei-c.org/ns/1.0" xmlns="http://chs.harvard.edu/xmlns/cts"><request><requestName>GetPassage</requestName><requestUrn>urn:cts:latinLit:phi0448.phi001.perseus-lat2:1.1.1-1.1.2</requestUrn></request><reply><urn>urn:cts:latinLit:phi0448.phi001.perseus-lat2:1.1.1-1.1.2</urn><passage><TEI xmlns="http://www.tei-c.org/ns/1.0"><text><body><div type="edition" xml:lang="lat" n="urn:cts:latinLit:phi0448.phi001.perseus-lat2"><div n="1" type="textpart" subtype="book"><div type="textpart" subtype="chapter" n="1"><div type="textpart" subtype="section" n="1"><p>Gallia est omnis divisa in partes tres, quarum unam incolunt Belgae, aliam Aquitani, tertiam qui ipsorum lingua Celtae, nostra Galli appellantur.</p></div><div type="textpart" subtype="section" n="2"><p>Hi omnes lingua, institutis, legibus inter se differunt. Gallos ab Aquitanis Garumna flumen, a Belgis Matrona et Sequana dividit.</p></div></div></div></div></body></text></TEI></passage></reply></GetPassage>'
cts_passage_xml_1_level: str = '<GetPassage xmlns:tei="http://www.tei-c.org/ns/1.0" xmlns="http://chs.harvard.edu/xmlns/cts"><request><requestName>GetPassage</requestName><requestUrn>urn:cts:latinLit:phi0448.phi001.perseus-lat2:1.1-1.2</requestUrn></request><reply><urn>urn:cts:latinLit:phi0448.phi001.perseus-lat2:1.1-1.2</urn><passage><TEI xmlns="http://www.tei-c.org/ns/1.0"><text><body><div type="edition" xml:lang="lat" n="urn:cts:latinLit:phi0448.phi001.perseus-lat2"><div n="1" type="textpart" subtype="book"><note>fake textual criticism</note><p>Gallia est omnis divisa in partes tres, quarum unam incolunt Belgae, aliam Aquitani, tertiam qui ipsorum lingua Celtae, nostra Galli appellantur.</p></div><div n="2" type="textpart" subtype="book"><p>Gallia est omnis divisa in partes tres, quarum unam incolunt Belgae, aliam Aquitani, tertiam qui ipsorum lingua Celtae, nostra Galli appellantur.</p></div><div n="3" type="textpart" subtype="book"><p>Gallia est omnis divisa in partes tres, quarum unam incolunt Belgae, aliam Aquitani, tertiam qui ipsorum lingua Celtae, nostra Galli appellantur.</p></div></div></body></text></TEI></passage></reply></GetPassage>'
diff --git a/mc_backend/openapi/openapi_server/models/corpus.py b/mc_backend/openapi/openapi_server/models/corpus.py
index b9a8470..e748eb0 100644
--- a/mc_backend/openapi/openapi_server/models/corpus.py
+++ b/mc_backend/openapi/openapi_server/models/corpus.py
@@ -112,6 +112,8 @@ class Corpus(Model):
:param cid: The cid of this Corpus.
:type cid: int
"""
+ if cid is None:
+ raise ValueError("Invalid value for `cid`, must not be `None`") # noqa: E501
self._cid = cid
diff --git a/mc_backend/openapi/openapi_server/openapi/openapi.yaml b/mc_backend/openapi/openapi_server/openapi/openapi.yaml
index bc97919..6d6749d 100644
--- a/mc_backend/openapi/openapi_server/openapi/openapi.yaml
+++ b/mc_backend/openapi/openapi_server/openapi/openapi.yaml
@@ -717,7 +717,6 @@ components:
required:
- plain_text
type: object
- x-body-name: raw_text_data
ZenodoMetadataPrefixExtension:
description: "Prefix for requests to the Zenodo API, indicating the desired\
\ metadata scheme."
@@ -750,7 +749,7 @@ components:
title: cid
type: integer
x-primary-key: true
- x-autoincrement: true
+ x-autoincrement: false
citation_level_1:
default: default
description: First level for citing the corpus.
@@ -783,6 +782,7 @@ components:
title: title
type: string
required:
+ - cid
- source_urn
title: Corpus
type: object
@@ -1327,7 +1327,6 @@ components:
- urn
type: object
type: object
- x-body-name: exercise_data
ExerciseBase:
description: Base data for creating and evaluating interactive exercises.
properties:
@@ -1512,7 +1511,6 @@ components:
example: urn:cts:latinLit:phi0448.phi001.perseus-lat2:1.1.1-1.1.1
type: string
type: object
- x-body-name: file_data
H5PForm:
description: Metadata for the H5P exercise.
properties:
@@ -1533,7 +1531,6 @@ components:
type: integer
type: array
type: object
- x-body-name: h5p_data
ExerciseTypePath:
description: Paths to the data directories for various H5P exercise types.
enum:
@@ -1576,7 +1573,6 @@ components:
- search_values
- urn
type: object
- x-body-name: kwic_data
RawTextForm:
description: Text data for an annotation request.
properties:
@@ -1587,7 +1583,6 @@ components:
required:
- plain_text
type: object
- x-body-name: raw_text_data
StaticExercise:
description: Metadata for a static exercise.
properties:
@@ -1631,7 +1626,6 @@ components:
required:
- search_regex
type: object
- x-body-name: network_data
Sentence:
description: Sentence with metadata
example:
@@ -1670,7 +1664,6 @@ components:
- query_urn
- vocabulary
type: object
- x-body-name: vocabulary_data
ZenodoRecord:
description: Record with header and metadata from Zenodo.
example:
@@ -1789,7 +1782,6 @@ components:
required:
- record_id
type: object
- x-body-name: zenodo_data
ExerciseAuthor:
description: Authors of curated exercises in the Machina Callida.
enum:
diff --git a/mc_backend/openapi_models.yaml b/mc_backend/openapi_models.yaml
index 90f1e05..092d647 100644
--- a/mc_backend/openapi_models.yaml
+++ b/mc_backend/openapi_models.yaml
@@ -95,7 +95,7 @@ components:
description: Unique identifier for the corpus.
example: 1
x-primary-key: true
- x-autoincrement: true
+ x-autoincrement: false
citation_level_1:
type: string
description: First level for citing the corpus.
@@ -123,6 +123,7 @@ components:
nullable: false
default: Anonymus
required:
+ - cid
- source_urn
Exercise:
allOf:
@@ -243,7 +244,6 @@ components:
- eid
- last_access_time
ExerciseForm:
- x-body-name: exercise_data
type: object
allOf:
- $ref: '#/components/schemas/ExerciseBase'
@@ -272,7 +272,6 @@ components:
description: Paths to the data directories for various H5P exercise types.
example: drag_text
FileForm:
- x-body-name: file_data
description: Metadata for file content to be saved for later access.
type: object
properties:
@@ -348,7 +347,6 @@ components:
H5PForm:
type: object
description: Metadata for the H5P exercise.
- x-body-name: h5p_data
properties:
eid:
type: string
@@ -367,7 +365,6 @@ components:
example: 0
description: Indices for the solutions that should be included in the download.
KwicForm:
- x-body-name: kwic_data
type: object
description: Relevant parameters for creating a Keyword In Context view.
properties:
@@ -610,7 +607,6 @@ components:
description: "Linguistic phenomena: syntactic dependencies, morphological features, lemmata, parts of speech."
example: upostag
RawTextForm:
- x-body-name: raw_text_data
description: Text data for an annotation request.
properties:
plain_text:
@@ -784,7 +780,6 @@ components:
- last_modified_time
- resource_type
VectorNetworkForm:
- x-body-name: network_data
type: object
description: Relevant parameters for finding sentences that are similar to a target word.
properties:
@@ -801,7 +796,6 @@ components:
PROIEL treebank, Vischer Wortkunde, VIVA textbook"
example: agldt
VocabularyForm:
- x-body-name: vocabulary_data
type: object
description: Relevant parameters for comparing a corpus to a reference vocabulary.
properties:
@@ -820,7 +814,6 @@ components:
- query_urn
- vocabulary
ZenodoForm:
- x-body-name: zenodo_data
type: object
description: Relevant parameters for accessing file URIs in a Zenodo record.
properties:
diff --git a/mc_backend/requirements.txt b/mc_backend/requirements.txt
index 22df72c..f0ebac8 100644
--- a/mc_backend/requirements.txt
+++ b/mc_backend/requirements.txt
@@ -1,39 +1,51 @@
-alembic==1.4.2
+a2wsgi==1.10.0
+alembic==1.13.1
aniso8601==9.0.1
-attrs==20.3.0
+anyio==4.2.0
+asgiref==3.7.2
+attrs==23.2.0
beautifulsoup4==4.9.0
blinker==1.6.3
cachetools==4.2.1
certifi==2020.12.5
cffi==1.14.5
chardet==3.0.4
+charset-normalizer==3.3.2
click==8.1.7
clickclick==20.10.2
conllu==2.3.2
-connexion==2.14.2
+connexion==3.0.6
coverage==5.5
cycler==0.10.0
decorator==4.4.2
-Flask==2.2.5
-Flask-Cors==3.0.10
-Flask-Migrate==2.7.0
+exceptiongroup==1.2.0
+Flask==3.0.1
+Flask-Migrate==4.0.5
Flask-RESTful==0.3.10
-Flask-SQLAlchemy==2.5.1
+Flask-SQLAlchemy==3.1.1
frozendict==1.2
future==0.18.2
gensim==3.8.2
graphannis==0.27.0
greenlet==3.0.1
gunicorn==20.0.4
+h11==0.14.0
html5lib==1.1
+httpcore==1.0.2
+httptools==0.6.1
+httpx==0.26.0
idna==2.10
importlib-metadata==6.8.0
+importlib-resources==5.13.0
inflection==0.5.1
isodate==0.6.0
itsdangerous==2.1.2
Jinja2==3.1.2
-jsonschema==3.2.0
+jsonschema==4.17.3
+jsonschema-spec==0.1.6
+jsonschema-specifications==2023.12.1
kiwisolver==1.3.1
+lazy-object-proxy==1.10.0
LinkHeader==0.4.3
lxml==4.6.2
Mako==1.1.4
@@ -44,12 +56,14 @@ mypy==0.812
mypy-extensions==0.4.3
networkx==3.1
numpy==1.24.4
-openalchemy==2.5.0
-openapi-schema-validator==0.1.4
-openapi-spec-validator==0.3.0
+OpenAlchemy==2.0.1
+openapi-schema-validator==0.4.4
+openapi-spec-validator==0.5.7
packaging==23.2
+pathable==0.4.3
Pillow==8.1.2
-prance==0.18.2
+pkgutil_resolve_name==1.3.10
+prance==23.6.21.0
psycopg2-binary==2.8.6
pycparser==2.20
PyLD==2.0.3
@@ -60,28 +74,40 @@ python-dateutil==2.8.1
python-docx==0.8.10
python-dotenv==0.13.0
python-editor==1.0.4
+python-multipart==0.0.6
python-rapidjson==1.0
pytz==2021.1
PyYAML==5.4.1
rdflib==6.0.1
rdflib-jsonld==0.6.2
+referencing==0.33.0
reportlab==3.5.63
-requests==2.23.0
+requests==2.31.0
+rfc3339-validator==0.1.4
+rpds-py==0.17.1
+ruamel.yaml==0.18.5
+ruamel.yaml.clib==0.2.8
scipy==1.6.1
semver==2.13.0
Sickle==0.7.0
-six==1.14.0
+six==1.16.0
smart-open==6.3.0
+sniffio==1.3.0
soupsieve==2.2
-SQLAlchemy==1.4.49
+SQLAlchemy==2.0.25
sqlalchemy-stubs==0.4
SQLAlchemy-Utils==0.41.1
-swagger-ui-bundle==0.0.8
+starlette==0.37.1
+swagger_ui_bundle==1.1.0
typed-ast==1.4.2
typing==3.7.4.1
typing_extensions==4.8.0
urllib3==1.25.11
+uvicorn==0.27.1
+uvloop==0.19.0
+watchfiles==0.21.0
webencodings==0.5.1
-Werkzeug==2.2.3
+websockets==12.0
+Werkzeug==3.0.1
xhtml2pdf==0.2.4
zipp==3.17.0
diff --git a/mc_backend/tests.py b/mc_backend/tests.py
index d47f71d..c89a64a 100644
--- a/mc_backend/tests.py
+++ b/mc_backend/tests.py
@@ -17,18 +17,21 @@ from unittest.mock import patch, MagicMock, mock_open
from zipfile import ZipFile
import MyCapytain
+import connexion
+import flask
import rapidjson as json
+import sqlalchemy
from conllu import TokenList
from flask import Flask
from gensim.models import Word2Vec
from graphannis.errors import NoSuchCorpus
+from httpx import Response
from lxml import etree
from networkx import MultiDiGraph, Graph
from requests import HTTPError
from sqlalchemy import inspect
from sqlalchemy.exc import OperationalError, InvalidRequestError
from sqlalchemy.orm import session
-from werkzeug.wrappers import Response
import mcserver
from mcserver.app import create_app, db, start_updater, full_init, log_exception, get_api_specification, \
@@ -86,16 +89,15 @@ class APItestCase(unittest.TestCase):
def test_api_corpus_delete(self):
""" Deletes a single corpus. """
- db.session.query(Corpus).delete()
- response: Response = Mocks.app_dict[self.class_name].client.delete(
- f"{Config.SERVER_URI_CORPORA}/1")
+ DatabaseService.delete(Corpus)
+ response: Response = Mocks.app_dict[self.class_name].client.delete(f"{Config.SERVER_URI_CORPORA}/1")
self.assertEqual(response.status_code, 404)
TestHelper.add_corpus(Mocks.corpora[0])
response = Mocks.app_dict[self.class_name].client.delete(f"{Config.SERVER_URI_CORPORA}/{Mocks.corpora[0].cid}")
- data_json: dict = json.loads(response.get_data())
+ data_json: bool = response.json()
self.assertEqual(data_json, True)
- db.session.query(Corpus).delete()
- db.session.query(UpdateInfo).delete()
+ DatabaseService.delete(Corpus)
+ DatabaseService.delete(UpdateInfo)
# dirty hack so we can reuse it in other tests
session.make_transient(Mocks.corpora[0])
@@ -105,13 +107,12 @@ class APItestCase(unittest.TestCase):
f"{Config.SERVER_URI_CORPORA}/{Mocks.corpora[0].cid}")
self.assertEqual(response.status_code, 404)
TestHelper.add_corpus(Mocks.corpora[0])
- response: Response = Mocks.app_dict[self.class_name].client.get(
- f"{Config.SERVER_URI_CORPORA}/{Mocks.corpora[0].cid}")
- data_json: dict = json.loads(response.get_data())
+ response = Mocks.app_dict[self.class_name].client.get(f"{Config.SERVER_URI_CORPORA}/{Mocks.corpora[0].cid}")
+ data_json: dict = response.json()
old_dict: dict = Mocks.corpora[0].to_dict()
self.assertEqual(data_json, old_dict)
- db.session.query(Corpus).delete()
- db.session.query(UpdateInfo).delete()
+ DatabaseService.delete(Corpus)
+ DatabaseService.delete(UpdateInfo)
# dirty hack so we can reuse it in other tests
session.make_transient(Mocks.corpora[0])
@@ -124,8 +125,8 @@ class APItestCase(unittest.TestCase):
last_modified_time=lmt.timestamp(), created_time=1.0)
mock.session.query.return_value = MockQuery(ui)
response: Response = Mocks.app_dict[self.class_name].client.get(
- TestingConfig.SERVER_URI_CORPORA, query_string=dict(last_update_time=lut))
- data_json = json.loads(response.get_data())
+ TestingConfig.SERVER_URI_CORPORA, params=dict(last_update_time=lut))
+ data_json = response.json()
if data_json:
result = [x.to_dict() for x in result]
self.assertEqual(data_json, result)
@@ -136,8 +137,8 @@ class APItestCase(unittest.TestCase):
db.session.add_all(Mocks.corpora)
DatabaseService.commit()
expect_result(self, mock_db, "0", Mocks.corpora, datetime.fromtimestamp(time.time()))
- db.session.query(Corpus).delete()
- db.session.query(UpdateInfo).delete()
+ DatabaseService.delete(Corpus)
+ DatabaseService.delete(UpdateInfo)
# dirty hack so we can reuse it in other tests
session.make_transient(Mocks.corpora[0])
@@ -149,23 +150,23 @@ class APItestCase(unittest.TestCase):
TestHelper.add_corpus(Mocks.corpora[0])
old_title: str = Mocks.corpora[0].title
new_title: str = "new_title"
- response: Response = Mocks.app_dict[self.class_name].client.patch(
+ response = Mocks.app_dict[self.class_name].client.patch(
f"{Config.SERVER_URI_CORPORA}/{Mocks.corpora[0].cid}", data=dict(title=new_title))
- data_json: dict = json.loads(response.get_data())
+ data_json: dict = response.json()
old_dict: dict = Mocks.corpora[0].to_dict()
self.assertEqual(data_json["title"], old_dict["title"])
Mocks.corpora[0].title = old_title
- db.session.query(Corpus).delete()
- db.session.query(UpdateInfo).delete()
+ DatabaseService.delete(Corpus)
+ DatabaseService.delete(UpdateInfo)
# dirty hack so we can reuse it in other tests
session.make_transient(Mocks.corpora[0])
def test_api_exercise_get(self):
""" Retrieves an existing exercise by its exercise ID. """
ar_copy: AnnisResponse = Mocks.copy(Mocks.annis_response, AnnisResponse)
- db.session.query(Exercise).delete()
+ DatabaseService.delete(Exercise)
response: Response = Mocks.app_dict[self.class_name].client.get(
- Config.SERVER_URI_EXERCISE, query_string=dict(eid=""))
+ Config.SERVER_URI_EXERCISE, params=dict(eid=""))
self.assertEqual(response.status_code, 404)
old_urn: str = Mocks.exercise.urn
Mocks.exercise.urn = ""
@@ -174,15 +175,15 @@ class APItestCase(unittest.TestCase):
mock_ar: AnnisResponse = AnnisResponse(solutions=[], graph_data=GraphData(links=[], nodes=[]))
with patch.object(CorpusService, "get_corpus", side_effect=[mock_ar, ar_copy]):
response = Mocks.app_dict[self.class_name].client.get(Config.SERVER_URI_EXERCISE,
- query_string=dict(eid=Mocks.exercise.eid))
+ params=dict(eid=Mocks.exercise.eid))
self.assertEqual(response.status_code, 404)
Mocks.exercise.urn = old_urn
DatabaseService.commit()
response = Mocks.app_dict[self.class_name].client.get(Config.SERVER_URI_EXERCISE,
- query_string=dict(eid=Mocks.exercise.eid))
- ar: AnnisResponse = AnnisResponse.from_dict(json.loads(response.get_data(as_text=True)))
+ params=dict(eid=Mocks.exercise.eid))
+ ar: AnnisResponse = AnnisResponse.from_dict(response.json())
self.assertEqual(len(ar.graph_data.nodes), 52)
- db.session.query(Exercise).delete()
+ DatabaseService.delete(Exercise)
session.make_transient(Mocks.exercise)
def test_api_exercise_list_get(self):
@@ -193,29 +194,29 @@ class APItestCase(unittest.TestCase):
DatabaseService.commit()
args: dict = dict(lang="fr", last_update_time=int(time.time()))
response: Response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_EXERCISE_LIST,
- query_string=args)
- self.assertEqual(json.loads(response.get_data()), [])
+ params=args)
+ self.assertEqual(response.json(), [])
args["last_update_time"] = 0
db.session.add(Mocks.exercise)
DatabaseService.commit()
- response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_EXERCISE_LIST, query_string=args)
+ response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_EXERCISE_LIST, params=args)
exercises: List[MatchingExercise] = []
- for exercise_dict in json.loads(response.get_data(as_text=True)):
+ for exercise_dict in response.json():
exercise_dict["search_values"] = json.dumps(exercise_dict["search_values"])
exercise_dict["solutions"] = json.dumps(exercise_dict["solutions"])
exercises.append(MatchingExercise.from_dict(exercise_dict))
self.assertEqual(len(exercises), 1)
args = dict(lang=Language.English.value, vocabulary=VocabularyCorpus.agldt.name, frequency_upper_bound=500)
- response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_EXERCISE_LIST, query_string=args)
- exercises: List[dict] = json.loads(response.get_data())
+ response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_EXERCISE_LIST, params=args)
+ exercises: List[dict] = response.json()
self.assertTrue(exercises[0]["matching_degree"])
- db.session.query(Exercise).delete()
- db.session.query(UpdateInfo).delete()
+ DatabaseService.delete(Exercise)
+ DatabaseService.delete(UpdateInfo)
session.make_transient(Mocks.exercise)
def test_api_exercise_post(self):
""" Creates a new exercise from scratch. """
- db.session.query(UpdateInfo).delete()
+ DatabaseService.delete(UpdateInfo)
ui_exercises: UpdateInfo = UpdateInfo.from_dict(resource_type=ResourceType.exercise_list.name,
last_modified_time=1.0, created_time=1.0)
db.session.add(ui_exercises)
@@ -230,13 +231,13 @@ class APItestCase(unittest.TestCase):
"text_complexity", return_value=Mocks.text_complexity):
response: Response = Mocks.app_dict[self.class_name].client.post(
Config.SERVER_URI_EXERCISE, headers=Mocks.headers_form_data, data=ef.to_dict())
- ar: AnnisResponse = AnnisResponse.from_dict(json.loads(response.get_data(as_text=True)))
+ ar: AnnisResponse = AnnisResponse.from_dict(response.json())
self.assertEqual(len(ar.solutions), 3)
response = Mocks.app_dict[self.class_name].client.post(
Config.SERVER_URI_EXERCISE, headers=Mocks.headers_form_data, data=ef.to_dict())
self.assertEqual(response.status_code, 500)
Mocks.app_dict[self.class_name].app_context.push()
- db.session.query(UpdateInfo).delete()
+ DatabaseService.delete(UpdateInfo)
def test_api_file_get(self):
"""Gets an existing exercise"""
@@ -248,26 +249,26 @@ class APItestCase(unittest.TestCase):
FileService.create_tmp_file(FileType.XML, "old")
args: dict = dict(type=FileType.XML, id=Mocks.exercise.eid, solution_indices=[0])
response: Response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_FILE,
- query_string=args)
+ params=args)
self.assertEqual(response.status_code, 404)
- file_path: str = os.path.join(Config.TMP_DIRECTORY, Mocks.exercise.eid + "." + FileType.XML)
+ file_path: str = str(os.path.join(Config.TMP_DIRECTORY, Mocks.exercise.eid + "." + FileType.XML))
file_content: str = "<xml></xml>"
with open(file_path, "w+") as f:
f.write(file_content)
ui_file.last_modified_time = datetime.utcnow().timestamp()
DatabaseService.commit()
del ui_file
- response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_FILE, query_string=args)
+ response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_FILE, params=args)
os.remove(file_path)
- self.assertEqual(response.get_data(as_text=True), file_content)
+ self.assertEqual(response.content.decode("utf-8"), file_content)
# add the mapped exercise to the database
db.session.add(Mocks.exercise)
DatabaseService.commit()
args["type"] = FileType.PDF
- response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_FILE, query_string=args)
- # the PDFs are not deterministically reproducible because the creation date etc. is written into them
- self.assertTrue(response.data.startswith(Mocks.exercise_pdf))
- db.session.query(Exercise).delete()
+ response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_FILE, params=args)
+ # the PDFs are not deterministically reproducible because the creation date, etc. is written into them
+ self.assertTrue(response.content.startswith(Mocks.exercise_pdf))
+ DatabaseService.delete(Exercise)
session.make_transient(Mocks.exercise)
def test_api_file_post(self):
@@ -280,10 +281,10 @@ class APItestCase(unittest.TestCase):
data_dict: dict = dict(file_type=FileType.XML, urn=Mocks.urn_custom, html_content="<html></html>")
response: Response = Mocks.app_dict[self.class_name].client.post(
TestingConfig.SERVER_URI_FILE, headers=Mocks.headers_form_data, data=data_dict)
- file_name = json.loads(response.data.decode("utf-8"))
+ file_name = response.json()
self.assertTrue(file_name.endswith(".xml"))
os.remove(os.path.join(Config.TMP_DIRECTORY, file_name))
- LearningResult.query.delete()
+ DatabaseService.delete(LearningResult)
def test_api_frequency_get(self):
""" Requests a frequency analysis for a given URN. """
@@ -294,13 +295,13 @@ class APItestCase(unittest.TestCase):
mcserver.app.services.corpusService.CorpusService, "get_frequency_analysis",
side_effect=[[FrequencyItem(values=[], phenomena=[], count=0)], expected_fa]):
response: Response = Mocks.app_dict[self.class_name].client.get(
- TestingConfig.SERVER_URI_FREQUENCY, query_string=dict(urn=Mocks.urn_custom))
- result_list: List[dict] = json.loads(response.get_data(as_text=True))
+ TestingConfig.SERVER_URI_FREQUENCY, params=dict(urn=Mocks.urn_custom))
+ result_list: List[dict] = response.json()
fa: List[FrequencyItem] = [FrequencyItem.from_dict(x) for x in result_list]
self.assertEqual(len(fa), 1)
response = Mocks.app_dict[self.class_name].client.get(
- TestingConfig.SERVER_URI_FREQUENCY, query_string=dict(urn=Mocks.urn_custom))
- result_list: List[dict] = json.loads(response.get_data(as_text=True))
+ TestingConfig.SERVER_URI_FREQUENCY, params=dict(urn=Mocks.urn_custom))
+ result_list: List[dict] = response.json()
fa: List[FrequencyItem] = [FrequencyItem.from_dict(x) for x in result_list]
self.assertEqual(fa[0].values, expected_fa[0].values)
self.assertEqual(fa[1].values[0], None)
@@ -308,33 +309,33 @@ class APItestCase(unittest.TestCase):
def test_api_h5p_get(self):
""" Requests a H5P JSON file for a given exercise. """
response: Response = Mocks.app_dict[self.class_name].client.get(
- TestingConfig.SERVER_URI_H5P, query_string=dict(eid=Config.EXERCISE_ID_TEST, lang=Language.English.value))
- self.assertIn(Mocks.h5p_json_cloze[1:-1], response.get_data(as_text=True))
+ TestingConfig.SERVER_URI_H5P, params=dict(eid=Config.EXERCISE_ID_TEST, lang=Language.English.value))
+ self.assertIn(Mocks.h5p_json_cloze[1:-1], response.content.decode("utf-8"))
args: dict = dict(eid=Mocks.exercise.eid, lang=Language.English.value, solution_indices=[0])
- response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_H5P, query_string=args)
+ response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_H5P, params=args)
self.assertEqual(response.status_code, 404)
db.session.add(Mocks.exercise)
DatabaseService.commit()
- response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_H5P, query_string=args)
- self.assertIn(Mocks.h5p_json_cloze[1:-1], response.get_data(as_text=True))
+ response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_H5P, params=args)
+ self.assertIn(Mocks.h5p_json_cloze[1:-1], response.content.decode("utf-8"))
Mocks.exercise.exercise_type = ExerciseType.kwic.value
DatabaseService.commit()
- response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_H5P, query_string=args)
+ response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_H5P, params=args)
self.assertEqual(response.status_code, 422)
Mocks.exercise.exercise_type = ExerciseType.matching.value
DatabaseService.commit()
- response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_H5P, query_string=args)
- self.assertIn(Mocks.h5p_json_matching[1:-1], response.get_data(as_text=True))
+ response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_H5P, params=args)
+ self.assertIn(Mocks.h5p_json_matching[1:-1], response.content.decode("utf-8"))
Mocks.exercise.exercise_type = ExerciseType.cloze.value
args["lang"] = "fr"
- response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_H5P, query_string=args)
- self.assertIn(Mocks.h5p_json_cloze[1:-1], response.get_data(as_text=True))
- db.session.query(Exercise).delete()
+ response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_H5P, params=args)
+ self.assertIn(Mocks.h5p_json_cloze[1:-1], response.content.decode("utf-8"))
+ DatabaseService.delete(Exercise)
session.make_transient(Mocks.exercise)
response = NetworkService.make_json_response(dict())
with patch.object(mcserver.app.api.h5pAPI, "get_remote_exercise", return_value=response) as get_mock:
args["eid"] = f".{FileType.H5P}"
- Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_H5P, query_string=args)
+ Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_H5P, params=args)
self.assertEqual(get_mock.call_count, 1)
def test_api_h5p_post(self):
@@ -349,12 +350,12 @@ class APItestCase(unittest.TestCase):
DatabaseService.commit()
response = Mocks.app_dict[self.class_name].client.post(
TestingConfig.SERVER_URI_H5P, headers=Mocks.headers_form_data, data=hf.to_dict())
- self.assertEqual(len(response.get_data()), 1940145)
+ self.assertEqual(len(response.content), 1940145)
with patch.object(mcserver.app.api.h5pAPI, "get_text_field_content", return_value=""):
response = Mocks.app_dict[self.class_name].client.post(
TestingConfig.SERVER_URI_H5P, headers=Mocks.headers_form_data, data=hf.to_dict())
self.assertEqual(response.status_code, 422)
- db.session.query(Exercise).delete()
+ DatabaseService.delete(Exercise)
def test_api_kwic_post(self):
""" Posts an AQL query to create a KWIC visualization in SVG format. """
@@ -372,7 +373,7 @@ class APItestCase(unittest.TestCase):
with patch.object(mcserver.app.services.corpusService.requests, "post", return_value=mr):
response: Response = Mocks.app_dict[self.class_name].client.post(
TestingConfig.SERVER_URI_KWIC, headers=Mocks.headers_form_data, data=kf.to_dict())
- self.assertTrue(response.data.startswith(Mocks.kwic_svg))
+ self.assertTrue(response.content.startswith(Mocks.kwic_svg))
def test_api_not_found(self):
"""Checks the 404 response in case of an invalid API query URL."""
@@ -387,18 +388,18 @@ class APItestCase(unittest.TestCase):
mock_get_corpus.return_value = AnnisResponse(
graph_data=GraphData(links=[], nodes=[]), solutions=[])
response: Response = Mocks.app_dict[self.class_name].client.get(
- TestingConfig.SERVER_URI_RAW_TEXT, query_string=dict(urn=Mocks.urn_custom))
+ TestingConfig.SERVER_URI_RAW_TEXT, params=dict(urn=Mocks.urn_custom))
self.assertEqual(response.status_code, 404)
mock_get_corpus.return_value = Mocks.annis_response
response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_RAW_TEXT,
- query_string=dict(urn=Mocks.urn_custom))
- ar: AnnisResponse = AnnisResponse.from_dict(json.loads(response.get_data(as_text=True)))
+ params=dict(urn=Mocks.urn_custom))
+ ar: AnnisResponse = AnnisResponse.from_dict(response.json())
self.assertEqual(len(ar.graph_data.nodes), 52)
ar_copy: AnnisResponse = Mocks.copy(Mocks.annis_response, AnnisResponse)
ar_copy.graph_data.nodes = []
mock_get_corpus.return_value = ar_copy
response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_RAW_TEXT,
- query_string=dict(urn=Mocks.urn_custom))
+ params=dict(urn=Mocks.urn_custom))
self.assertEqual(response.status_code, 404)
def test_api_raw_text_post(self):
@@ -407,7 +408,7 @@ class APItestCase(unittest.TestCase):
with patch.object(AnnotationService, "get_udpipe", return_value=Mocks.udpipe_string):
response: Response = Mocks.app_dict[self.class_name].client.post(
TestingConfig.SERVER_URI_RAW_TEXT, headers=Mocks.headers_form_data, data=rtfe.to_dict())
- ar: AnnisResponse = AnnisResponse.from_dict(json.loads(response.get_data()))
+ ar: AnnisResponse = AnnisResponse.from_dict(response.json())
self.assertEqual(len(ar.graph_data.nodes), 5)
def test_api_static_exercises_get(self):
@@ -447,7 +448,7 @@ class APItestCase(unittest.TestCase):
self.assertEqual(response.status_code, 503)
response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_STATIC_EXERCISES)
os.remove(TestingConfig.STATIC_EXERCISES_ZIP_FILE_PATH)
- self.assertGreater(len(response.get_data(as_text=True)), 1900)
+ self.assertGreater(len(response.content.decode("utf-8")), 1900)
response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_STATIC_EXERCISES)
self.assertEqual(mock_udpipe.call_count, 1)
@@ -462,25 +463,25 @@ class APItestCase(unittest.TestCase):
"text_complexity", return_value=Mocks.text_complexity):
args: dict = dict(urn=Mocks.urn_custom, measure=TextComplexityMeasure.all.name)
response: Response = Mocks.app_dict[self.class_name].client.get(
- TestingConfig.SERVER_URI_TEXT_COMPLEXITY, query_string=args)
- self.assertEqual(response.get_data(as_text=True), json.dumps(Mocks.text_complexity.to_dict()))
+ TestingConfig.SERVER_URI_TEXT_COMPLEXITY, params=args)
+ self.assertEqual(response.json(), Mocks.text_complexity.to_dict())
def test_api_valid_reff_get(self): #
""" Retrieves possible citations for a given URN. """
with patch.object(MyCapytain.retrievers.cts5.requests, "get", side_effect=TestHelper.cts_get_mock):
args: dict = dict(urn=Mocks.urn_custom[:-14])
response: Response = Mocks.app_dict[self.class_name].client.get(
- TestingConfig.SERVER_URI_VALID_REFF, query_string=args)
- self.assertEqual(len(json.loads(response.data.decode("utf-8"))), 3)
+ TestingConfig.SERVER_URI_VALID_REFF, params=args)
+ self.assertEqual(len(response.json()), 3)
APItestCase.clear_folder(Config.REFF_CACHE_DIRECTORY)
args["urn"] = f"{Mocks.urn_custom[:-13]}4"
response = Mocks.app_dict[self.class_name].client.get(
- TestingConfig.SERVER_URI_VALID_REFF, query_string=args)
+ TestingConfig.SERVER_URI_VALID_REFF, params=args)
self.assertEqual(response.status_code, 404)
APItestCase.clear_folder(Config.REFF_CACHE_DIRECTORY)
args["urn"] = f"{Mocks.urn_custom[:-13]}abc"
response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_VALID_REFF,
- query_string=args)
+ params=args)
self.assertEqual(response.status_code, 400)
APItestCase.clear_folder(Config.REFF_CACHE_DIRECTORY)
TestingConfig.SIMULATE_HTTP_ERROR = True
@@ -495,8 +496,8 @@ class APItestCase(unittest.TestCase):
with patch.object(mcserver.app.api.vectorNetworkAPI.Word2Vec, "load", return_value=MockW2V()):
args: dict = dict(search_regex='ueritas', nearest_neighbor_count=150, min_count=6)
response: Response = Mocks.app_dict[self.class_name].client.get(
- TestingConfig.SERVER_URI_VECTOR_NETWORK, query_string=args)
- svg_string: str = json.loads(response.get_data(as_text=True))
+ TestingConfig.SERVER_URI_VECTOR_NETWORK, params=args)
+ svg_string: str = response.json()
self.assertGreater(len(svg_string), 6500)
def test_api_vector_network_post(self):
@@ -507,32 +508,34 @@ class APItestCase(unittest.TestCase):
vnf: VectorNetworkForm = VectorNetworkForm(search_regex='uera', nearest_neighbor_count=10)
response: Response = Mocks.app_dict[self.class_name].client.post(
TestingConfig.SERVER_URI_VECTOR_NETWORK, headers=Mocks.headers_form_data, data=vnf.to_dict())
- self.assertEqual(len(json.loads(response.get_data(as_text=True))), 2)
+ self.assertEqual(len(response.json()), 2)
def test_api_vocabulary_get(self):
""" Retrieves sentence ID and matching degree for each sentence in the query text. """
vf: VocabularyForm = VocabularyForm(query_urn="", vocabulary=VocabularyMC.AGLDT, frequency_upper_bound=6000)
response: Response = Mocks.app_dict[self.class_name].client.get(
- TestingConfig.SERVER_URI_VOCABULARY, query_string=vf.to_dict())
+ TestingConfig.SERVER_URI_VOCABULARY, params=vf.to_dict())
self.assertEqual(response.status_code, 404)
with patch.object(mcserver.app.services.CorpusService, "get_corpus",
return_value=Mocks.copy(Mocks.annis_response, AnnisResponse)):
- vf.query_urn=Mocks.urn_custom
+ vf.query_urn = Mocks.urn_custom
response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_VOCABULARY,
- query_string=vf.to_dict())
- sentences: List[Sentence] = [Sentence.from_dict(x) for x in json.loads(response.get_data(as_text=True))]
+ params=vf.to_dict())
+ sentences: List[Sentence] = [Sentence.from_dict(x) for x in response.json()]
self.assertEqual(sentences[0].matching_degree, 90.9090909090909)
def test_api_vocabulary_post(self):
""" Indicates for each token of a corpus whether it is covered by a reference vocabulary. """
with patch.object(mcserver.app.services.corpusService.CorpusService, "get_corpus",
return_value=Mocks.annis_response):
- vf: VocabularyForm = VocabularyForm(frequency_upper_bound=500, query_urn=Mocks.urn_custom,
- vocabulary=VocabularyMC.AGLDT)
- response: Response = Mocks.app_dict[self.class_name].client.post(
- TestingConfig.SERVER_URI_VOCABULARY, data=vf.to_dict())
- ar: AnnisResponse = AnnisResponse.from_dict(json.loads(response.get_data(as_text=True)))
- self.assertTrue(NodeMC.from_dict(ar.graph_data.nodes[3].to_dict()).is_oov)
+ with patch.object(mcserver.app.services.textComplexityService.TextComplexityService,
+ "text_complexity", return_value=Mocks.text_complexity):
+ vf: VocabularyForm = VocabularyForm(frequency_upper_bound=500, query_urn=Mocks.urn_custom,
+ vocabulary=VocabularyMC.AGLDT)
+ response: Response = Mocks.app_dict[self.class_name].client.post(
+ TestingConfig.SERVER_URI_VOCABULARY, data=vf.to_dict())
+ ar: AnnisResponse = AnnisResponse.from_dict(response.json())
+ self.assertTrue(NodeMC.from_dict(ar.graph_data.nodes[3].to_dict()).is_oov)
def test_api_zenodo_get(self):
""" Provides exercise materials from the Zenodo repository."""
@@ -541,8 +544,7 @@ class APItestCase(unittest.TestCase):
mock_records[0].metadata = dict(title=[title])
with patch.object(mcserver.app.api.zenodoAPI.Sickle, "ListRecords", return_value=mock_records):
response: Response = Mocks.app_dict[self.class_name].client.get(TestingConfig.SERVER_URI_ZENODO)
- records: List[ZenodoRecord] = [ZenodoRecord.from_dict(x) for x in
- json.loads(response.get_data(as_text=True))]
+ records: List[ZenodoRecord] = [ZenodoRecord.from_dict(x) for x in response.json()]
self.assertEqual(records[0].title, [title])
def test_api_zenodo_post(self):
@@ -553,12 +555,12 @@ class APItestCase(unittest.TestCase):
with patch.object(mcserver.app.api.zenodoAPI.Sickle, "GetRecord", return_value=record_mock):
response: Response = Mocks.app_dict[self.class_name].client.post(
TestingConfig.SERVER_URI_ZENODO, data=zf.to_dict())
- self.assertEqual(json.loads(response.get_data()), uris)
+ self.assertEqual(response.json(), uris)
def test_create_app(self):
"""Creates a new Flask application and configures it. Initializes the application and the database."""
with patch.object(sys, "argv", [None, None, Config.FLASK_MIGRATE]):
- with patch.object(mcserver.app, "init_app_common", return_value=Flask(__name__)):
+ with patch.object(mcserver.app, "init_app_common", return_value=connexion.FlaskApp(__name__)):
cfg: Type[Config] = TestingConfig
old_uri: str = cfg.SQLALCHEMY_DATABASE_URI
create_app(cfg)
@@ -571,7 +573,7 @@ class APItestCase(unittest.TestCase):
response: Response = Mocks.app_dict[self.class_name].client.get(Config.SERVER_URI_FAVICON)
with open(os.path.join(Config.ASSETS_DIRECTORY, Config.FAVICON_FILE_NAME), "rb") as f:
content: bytes = f.read()
- data_received: bytes = response.get_data()
+ data_received: bytes = response.content
self.assertEqual(content, data_received)
@@ -642,7 +644,7 @@ class CommonTestCase(unittest.TestCase):
db.session.add(ui_cts)
DatabaseService.commit()
clean_tmp_folder()
- db.session.query(UpdateInfo).delete()
+ DatabaseService.delete(UpdateInfo)
self.assertFalse(os.path.exists(dir_path))
def test_create_xml_string(self):
@@ -666,13 +668,13 @@ class CommonTestCase(unittest.TestCase):
def test_full_init(self):
""" Fully initializes the application, including logging."""
- Mocks.app_dict[self.class_name].app.config["TESTING"] = False
+ Mocks.app_dict[self.class_name].app.app.config["TESTING"] = False
with patch.object(CorpusService, "init_graphannis_logging"):
with patch.object(mcserver.app, "start_updater") as updater_mock:
- full_init(Mocks.app_dict[self.class_name].app)
+ full_init(Mocks.app_dict[self.class_name].app.app)
self.assertEqual(updater_mock.call_count, 1)
- Mocks.app_dict[self.class_name].app.config["TESTING"] = True
- db.session.query(UpdateInfo).delete()
+ Mocks.app_dict[self.class_name].app.app.config["TESTING"] = True
+ DatabaseService.delete(UpdateInfo)
def test_get_annotations_from_string(self):
""" Gets annotation data from a given string, be it a CoNLL string or a corpus URN. """
@@ -792,8 +794,8 @@ class CommonTestCase(unittest.TestCase):
response_mock.content = b""
with patch.object(mcserver.app.api.h5pAPI.requests, "get", return_value=response_mock):
with patch.object(mcserver.app.api.h5pAPI, "ZipFile", return_value=ZipMock):
- response: Response = get_remote_exercise("")
- h5p_dict: dict = json.loads(response.get_data())
+ response: flask.Response = get_remote_exercise("")
+ h5p_dict: dict = response.json
self.assertEqual(h5p_dict["mainLibrary"], "lib")
def test_get_solutions_by_index(self):
@@ -866,7 +868,6 @@ class CommonTestCase(unittest.TestCase):
import blinker # for signalling
import coverage # for code coverage in unit tests
from docx import Document # for exercise exports
- from flask_cors import CORS # for HTTP requests
from flask_migrate import Migrate # for database migrations
from flask_restful import Api # for construction of the API
import gunicorn # for production server
@@ -936,9 +937,9 @@ class CommonTestCase(unittest.TestCase):
def test_log_exception(self):
"""Logs errors that occur while the Flask app is working. """
- with patch.object(Mocks.app_dict[self.class_name].app.logger, "info") as mock_info:
- with Mocks.app_dict[self.class_name].app.test_request_context("/?param=value"):
- log_exception(Mocks.app_dict[self.class_name].app, ValueError())
+ with patch.object(Mocks.app_dict[self.class_name].app.app.logger, "info") as mock_info:
+ with Mocks.app_dict[self.class_name].app.app.test_request_context("/?param=value"):
+ log_exception(Mocks.app_dict[self.class_name].app.app, ValueError())
self.assertEqual(mock_info.call_count, 1)
def test_make_docx_file(self):
@@ -1031,7 +1032,7 @@ class CommonTestCase(unittest.TestCase):
self.assertEqual(len(xapi.serialize().keys()), 5)
ed: ExerciseData = ExerciseData(json_dict=Mocks.exercise_data.serialize())
self.assertEqual(len(ed.graph.nodes), len(Mocks.exercise_data.graph.nodes))
- db.session.query(UpdateInfo).delete()
+ DatabaseService.delete(UpdateInfo)
session.make_transient(Mocks.corpora[0])
session.make_transient(Mocks.exercise)
@@ -1068,11 +1069,11 @@ class CommonTestCase(unittest.TestCase):
db.session.add(ui_cts)
DatabaseService.commit()
with patch.object(CorpusService, 'update_corpora') as update_mock:
- t: Thread = start_updater(Mocks.app_dict[self.class_name].app)
+ t: Thread = start_updater(Mocks.app_dict[self.class_name].app.app)
self.assertIsInstance(t, Thread)
self.assertTrue(t.is_alive())
time.sleep(0.1)
- db.session.query(UpdateInfo).delete()
+ DatabaseService.delete(UpdateInfo)
assert not update_mock.called
Mocks.app_dict[self.class_name].app_context.push()
@@ -1105,8 +1106,8 @@ class CorpusTestCase(unittest.TestCase):
@patch('mcserver.app.services.corpusService.CorpusService.update_corpora')
def test_check_corpus_list_age(self, mock_update: MagicMock):
"""Checks whether the list of available corpora needs to be updated."""
- db.session.query(UpdateInfo).delete()
- CorpusService.check_corpus_list_age(Mocks.app_dict[self.class_name].app)
+ DatabaseService.delete(UpdateInfo)
+ CorpusService.check_corpus_list_age(Mocks.app_dict[self.class_name].app.app)
ui_cts: UpdateInfo = DatabaseService.query(
UpdateInfo, filter_by=dict(resource_type=ResourceType.cts_data.name), first=True)
self.assertEqual(ui_cts, None)
@@ -1115,11 +1116,11 @@ class CorpusTestCase(unittest.TestCase):
db.session.add(ui_cts)
DatabaseService.commit()
utc_now: datetime = datetime.utcnow()
- CorpusService.check_corpus_list_age(Mocks.app_dict[self.class_name].app)
+ CorpusService.check_corpus_list_age(Mocks.app_dict[self.class_name].app.app)
ui_cts = DatabaseService.query(
UpdateInfo, filter_by=dict(resource_type=ResourceType.cts_data.name), first=True)
self.assertGreater(ui_cts.last_modified_time, utc_now.timestamp())
- db.session.query(UpdateInfo).delete()
+ DatabaseService.delete(UpdateInfo)
def test_extract_custom_corpus_text(self):
""" Extracts text from the relevant parts of a (custom) corpus. """
@@ -1239,14 +1240,14 @@ class CorpusTestCase(unittest.TestCase):
ec: Corpus = CorpusService.existing_corpora[0]
ec.title = ""
DatabaseService.commit()
- TestHelper.add_corpus(CorpusMC.from_dict(source_urn="123"))
+ TestHelper.add_corpus(CorpusMC.from_dict(source_urn="123", cid=123))
cls: List[CitationLevel] = [ec.citation_level_1, ec.citation_level_2, ec.citation_level_3]
CorpusService.update_corpus(ec.title, ec.source_urn, ec.author, cls, ec)
self.assertFalse(ec.title)
CorpusService.update_corpora()
self.assertTrue(ec.title)
- db.session.query(Corpus).delete()
- db.session.query(UpdateInfo).delete()
+ DatabaseService.delete(Corpus)
+ DatabaseService.delete(UpdateInfo)
class DatabaseTestCase(unittest.TestCase):
@@ -1268,12 +1269,12 @@ class DatabaseTestCase(unittest.TestCase):
self.assertTrue(os.path.exists(Config.GRAPHANNIS_LOG_PATH))
os.remove(Config.GRAPHANNIS_LOG_PATH)
with patch.object(sys, 'argv', Mocks.test_args):
- app: Flask = mcserver.get_app()
+ app: Flask = mcserver.get_app().app
self.assertIsInstance(app, Flask)
self.assertTrue(app.config["TESTING"])
- db.session.query(UpdateInfo).delete()
+ DatabaseService.delete(UpdateInfo)
Mocks.app_dict[self.class_name].app_context.push()
- db.session.query(Corpus).delete()
+ DatabaseService.delete(Corpus)
def test_commit(self):
"""Commits the last action to the database and, if it fails, rolls back the current session."""
@@ -1285,30 +1286,32 @@ class DatabaseTestCase(unittest.TestCase):
mock_db.session.commit.side_effect = commit
with self.assertRaises(OperationalError):
TestHelper.add_corpus(Mocks.corpora[0])
- db.session.query(Corpus).delete()
- db.session.query(UpdateInfo).delete()
+ DatabaseService.delete(Corpus)
+ DatabaseService.delete(UpdateInfo)
session.make_transient(Mocks.corpora[0])
def test_create_postgres_database(self):
""" Creates a new Postgres database. """
with patch.object(mcserver.app.Session, "execute") as execute_mock:
with patch.object(mcserver.app.Session, "connection"):
- create_postgres_database(Mocks.app_dict[self.class_name].app, TestingConfig)
+ create_postgres_database(Mocks.app_dict[self.class_name].app.app, TestingConfig)
self.assertEqual(execute_mock.call_count, 1)
def test_init_db_alembic(self):
"""In Docker, the alembic version is not initially written to the database, so we need to set it manually."""
+ db.session.execute(sqlalchemy.text(f'DROP TABLE IF EXISTS {Config.DATABASE_TABLE_ALEMBIC}'))
+ DatabaseService.commit()
self.assertEqual(inspect(db.engine).has_table(Config.DATABASE_TABLE_ALEMBIC), False)
with patch.object(mcserver.app.services.databaseService, "flask_migrate") as migrate_mock:
migrate_mock.stamp.return_value = MagicMock()
migrate_mock.upgrade.return_value = MagicMock()
migrate_mock.current.return_value = MagicMock()
- DatabaseService.init_db_alembic(Mocks.app_dict[self.class_name].app)
+ DatabaseService.init_db_alembic(Mocks.app_dict[self.class_name].app.app)
self.assertEqual(migrate_mock.stamp.call_count, 1)
def test_init_db_corpus(self):
"""Initializes the corpus table."""
- db.session.query(Corpus).delete()
+ DatabaseService.delete(Corpus)
cc: CustomCorpus = CustomCorpusService.custom_corpora[0]
old_corpus: Corpus = Mocks.corpora[0]
old_corpus.source_urn = cc.corpus.source_urn
@@ -1318,8 +1321,8 @@ class DatabaseTestCase(unittest.TestCase):
corpus: Corpus = DatabaseService.query(
Corpus, filter_by=dict(source_urn=cc.corpus.source_urn), first=True)
self.assertEqual(corpus.title, cc.corpus.title)
- db.session.query(Corpus).delete()
- db.session.query(UpdateInfo).delete()
+ DatabaseService.delete(Corpus)
+ DatabaseService.delete(UpdateInfo)
def test_map_exercise_data_to_database(self):
"""Maps exercise data to the database and saves it for later access."""
@@ -1351,8 +1354,8 @@ class DatabaseTestCase(unittest.TestCase):
self.assertEqual(expected_values, actual_values)
exercise_from_db: Exercise = DatabaseService.query(Exercise, first=True)
self.assertEqual(exercise, exercise_from_db)
- db.session.query(Exercise).delete()
- db.session.query(UpdateInfo).delete()
+ DatabaseService.delete(Exercise)
+ DatabaseService.delete(UpdateInfo)
session.make_transient(Mocks.exercise)
def test_update_exercises(self):
@@ -1371,7 +1374,7 @@ class DatabaseTestCase(unittest.TestCase):
exercises = DatabaseService.query(Exercise)
self.assertEqual(len(exercises), 1)
self.assertEqual(exercises[0].text_complexity, 54.53)
- db.session.query(Exercise).delete()
+ DatabaseService.delete(Exercise)
if __name__ == '__main__':
diff --git a/mc_frontend/openapi/model/corpus.ts b/mc_frontend/openapi/model/corpus.ts
index f7fbe45..37629fe 100644
--- a/mc_frontend/openapi/model/corpus.ts
+++ b/mc_frontend/openapi/model/corpus.ts
@@ -22,7 +22,7 @@ export interface Corpus {
/**
* Unique identifier for the corpus.
*/
- cid?: number;
+ cid: number;
/**
* First level for citing the corpus.
*/
diff --git a/mc_frontend/package.json b/mc_frontend/package.json
index cf298b3..5a4d313 100644
--- a/mc_frontend/package.json
+++ b/mc_frontend/package.json
@@ -1,6 +1,6 @@
{
"name": "mc_frontend",
- "version": "2.9.1",
+ "version": "2.9.2",
"author": "Ionic Framework",
"homepage": "https://ionicframework.com/",
"scripts": {
diff --git a/mc_frontend/src/app/author-detail/author-detail.page.spec.ts b/mc_frontend/src/app/author-detail/author-detail.page.spec.ts
index b4962e7..18e8b96 100644
--- a/mc_frontend/src/app/author-detail/author-detail.page.spec.ts
+++ b/mc_frontend/src/app/author-detail/author-detail.page.spec.ts
@@ -42,7 +42,7 @@ describe('AuthorDetailPage', () => {
it('should show possible references', () => {
const currentCorpusSpy: Spy = spyOn(authorDetailPage.corpusService, 'setCurrentCorpus');
const textRangeSpy: Spy = spyOn(authorDetailPage.helperService, 'goToPage').and.returnValue(Promise.resolve(true));
- authorDetailPage.showPossibleReferences({source_urn: ''});
+ authorDetailPage.showPossibleReferences({source_urn: '', cid: -999});
expect(currentCorpusSpy).toHaveBeenCalledTimes(1);
expect(textRangeSpy).toHaveBeenCalledTimes(1);
});
diff --git a/mc_frontend/src/app/author/author.page.spec.ts b/mc_frontend/src/app/author/author.page.spec.ts
index 603643c..39b2d92 100644
--- a/mc_frontend/src/app/author/author.page.spec.ts
+++ b/mc_frontend/src/app/author/author.page.spec.ts
@@ -87,7 +87,7 @@ describe('AuthorPage', () => {
expect(toggleSpy).toHaveBeenCalledTimes(1);
expect(storageSpy).toHaveBeenCalledTimes(2);
authorPage.corpusService.availableAuthors = [new Author({
- corpora: [{source_urn: 'proiel'}],
+ corpora: [{source_urn: 'proiel', cid: -999}],
name: 'name'
})];
toggleSpy.and.callThrough();
diff --git a/mc_frontend/src/app/corpus.service.spec.ts b/mc_frontend/src/app/corpus.service.spec.ts
index 6d33c9d..f7ebff1 100644
--- a/mc_frontend/src/app/corpus.service.spec.ts
+++ b/mc_frontend/src/app/corpus.service.spec.ts
@@ -267,7 +267,7 @@ describe('CorpusService', () => {
it('should initialize the current corpus', ((done) => {
corpusService.helperService.applicationState.next(new ApplicationState({mostRecentSetup: new TextData()}));
- let corpus: CorpusMC = {source_urn: ''};
+ let corpus: CorpusMC = {source_urn: '', cid: -999};
const subscriptions: Subscription[] = [];
function initCorpus(): Promise<void> {
@@ -332,12 +332,12 @@ describe('CorpusService', () => {
it('should process corpora', () => {
const corpusList: CorpusMC[] = [
- {author: '', source_urn: 'proiel'},
- {author: 'b', source_urn: 'b', title: 't1'},
- {author: 'b', source_urn: 'b', title: 't1'},
- {author: 'b', source_urn: 'c', title: 't3'},
- {author: 'b', source_urn: 'e', title: 't2'},
- {author: 'a', source_urn: 'd'},
+ {author: '', source_urn: 'proiel', cid: -999},
+ {author: 'b', source_urn: 'b', title: 't1', cid: -999},
+ {author: 'b', source_urn: 'b', title: 't1', cid: -999},
+ {author: 'b', source_urn: 'c', title: 't3', cid: -999},
+ {author: 'b', source_urn: 'e', title: 't2', cid: -999},
+ {author: 'a', source_urn: 'd', cid: -999},
];
corpusService.availableAuthors = [new Author({name: ' PROIEL', corpora: []})];
corpusService.processCorpora(corpusList);
@@ -396,7 +396,7 @@ describe('CorpusService', () => {
it('should save a new corpus', (done) => {
corpusService.initCurrentTextRange();
corpusService.initCurrentCorpus().then(() => {
- corpusService.setCurrentCorpus({source_urn: ''});
+ corpusService.setCurrentCorpus({source_urn: '', cid: -999});
corpusService.currentCorpus.pipe(take(1)).subscribe((corpus: CorpusMC) => {
expect(corpus).toBeTruthy();
done();
diff --git a/mc_frontend/src/app/doc-software/doc-software.page.html b/mc_frontend/src/app/doc-software/doc-software.page.html
index 4849eb3..89ff5e4 100644
--- a/mc_frontend/src/app/doc-software/doc-software.page.html
+++ b/mc_frontend/src/app/doc-software/doc-software.page.html
@@ -30,7 +30,7 @@
</a>
</li>
<li>
- <a href="https://korpling.org/mc-service/mc/api/v1.0/ui/" style="text-decoration: none;" target="_blank">
+ <a href="https://korpling.org/mc-spec/" style="text-decoration: none;" target="_blank">
Documentation @ Swagger
</a>
</li>
diff --git a/mc_frontend/src/app/models/mockMC.ts b/mc_frontend/src/app/models/mockMC.ts
index b20033c..94ee054 100644
--- a/mc_frontend/src/app/models/mockMC.ts
+++ b/mc_frontend/src/app/models/mockMC.ts
@@ -17,6 +17,7 @@ export default class MockMC {
author: 'author',
source_urn: 'urn',
title: 'title',
+ cid: -999
}];
static apiResponseFrequencyAnalysisGet: FrequencyItem[] = [{
phenomena: [Phenomenon.Upostag],
@@ -35,7 +36,7 @@ export default class MockMC {
nodes: [{}], links: []
}
},
- currentCorpus: {citations: {}, source_urn: 'exampleUrn'},
+ currentCorpus: {citations: {}, source_urn: 'exampleUrn', cid: -999},
currentTextRange: new TextRange({start: ['1', '2'], end: ['1', '2']})
}),
exerciseList: [new ExerciseMC()],
diff --git a/mc_frontend/src/app/ranking/ranking.page.spec.ts b/mc_frontend/src/app/ranking/ranking.page.spec.ts
index 4aca635..781b030 100644
--- a/mc_frontend/src/app/ranking/ranking.page.spec.ts
+++ b/mc_frontend/src/app/ranking/ranking.page.spec.ts
@@ -64,15 +64,13 @@ describe('RankingPage', () => {
const oovSpy: Spy = spyOn(rankingPage.vocService, 'getOOVwords').and.returnValue(Promise.resolve(ar));
spyOn(rankingPage.corpusService, 'processAnnisResponse');
spyOn(rankingPage.helperService, 'goToShowTextPage').and.returnValue(Promise.resolve(true));
- rankingPage.showText([{id: 1}]).then(() => {
+ rankingPage.showText([{id: 1}]).then(async () => {
expect(rankingPage.helperService.isVocabularyCheck).toBe(true);
oovSpy.and.callFake(() => Promise.reject());
rankingPage.helperService.isVocabularyCheck = false;
- rankingPage.showText([{id: 1}]).then(() => {
- }, () => {
- expect(rankingPage.helperService.isVocabularyCheck).toBe(false);
- done();
- });
+ await rankingPage.showText([{id: 1}]);
+ expect(rankingPage.helperService.isVocabularyCheck).toBe(false);
+ done();
});
});
});
diff --git a/mc_frontend/src/app/ranking/ranking.page.ts b/mc_frontend/src/app/ranking/ranking.page.ts
index c65f200..53951e0 100644
--- a/mc_frontend/src/app/ranking/ranking.page.ts
+++ b/mc_frontend/src/app/ranking/ranking.page.ts
@@ -45,7 +45,7 @@ export class RankingPage implements OnInit {
this.corpusService.baseUrn = parsedUrnParts[0];
this.vocService.desiredSentenceCount = params.desiredSentenceCount;
this.vocService.frequencyUpperBound = params.frequencyUpperBound;
- this.corpusService.currentCorpus.next({source_urn: params.urn});
+ this.corpusService.currentCorpus.next({source_urn: params.urn, cid: -999});
this.corpusService.currentTextRange.next(parsedUrnParts[1]);
this.corpusService.isTextRangeCorrect = true;
this.vocService.currentReferenceVocabulary = params.referenceVocabulary;
@@ -54,7 +54,7 @@ export class RankingPage implements OnInit {
}
showText(rank: Sentence[]): Promise<void> {
- return new Promise<void>((resolve, reject) => {
+ return new Promise<void>((resolve) => {
this.corpusService.currentUrn = this.corpusService.baseUrn + `@${rank[0].id}-${rank[rank.length - 1].id}`;
this.vocService.getOOVwords(this.corpusService.currentUrn).then((ar: AnnisResponse) => {
const urnStart: string = ar.graph_data.nodes[0].id.split('/')[1];
@@ -65,7 +65,7 @@ export class RankingPage implements OnInit {
this.helperService.goToShowTextPage(this.navCtrl, true).then();
return resolve();
}, async (error: HttpErrorResponse) => {
- return reject();
+ return resolve();
});
});
}
diff --git a/mc_frontend/src/app/text-range/text-range.page.spec.ts b/mc_frontend/src/app/text-range/text-range.page.spec.ts
index 33e8a9b..8581c8b 100644
--- a/mc_frontend/src/app/text-range/text-range.page.spec.ts
+++ b/mc_frontend/src/app/text-range/text-range.page.spec.ts
@@ -51,12 +51,13 @@ describe('TextRangePage', () => {
it('should add level 3 references', (done) => {
const addReferencesSpy: Spy = spyOn(textRangePage, 'addReferences').and.returnValue(Promise.resolve());
- textRangePage.addLevel3References([], {source_urn: ''}).then(() => {
+ textRangePage.addLevel3References([], {source_urn: '', cid: -999}).then(() => {
expect(addReferencesSpy).toHaveBeenCalledTimes(0);
textRangePage.currentInputId = 2;
const corpus: CorpusMC = {
source_urn: '',
- citations: {1: new Citation({subcitations: {2: new Citation({subcitations: {3: new Citation()}})}})}
+ citations: {1: new Citation({subcitations: {2: new Citation({subcitations: {3: new Citation()}})}})},
+ cid: -999
};
textRangePage.addLevel3References(['1', '2'], corpus).then(() => {
expect(addReferencesSpy).toHaveBeenCalledTimes(0);
@@ -81,7 +82,7 @@ describe('TextRangePage', () => {
}
textRangePage.corpusService.currentCorpus = new ReplaySubject<CorpusMC>(1);
- textRangePage.corpusService.currentCorpus.next({citations: {4: new Citation()}, source_urn: ''});
+ textRangePage.corpusService.currentCorpus.next({citations: {4: new Citation()}, source_urn: '', cid: -999});
resetCitationValues();
const citationLabels: string[] = ['1'];
textRangePage.addMissingCitations(citationLabels, citationLabels).then(() => {
@@ -107,7 +108,11 @@ describe('TextRangePage', () => {
});
it('should add references', (done) => {
- const corpus: CorpusMC = {citations: {0: new Citation({subcitations: {}, label: ''})}, source_urn: ''};
+ const corpus: CorpusMC = {
+ citations: {0: new Citation({subcitations: {}, label: ''})},
+ source_urn: '',
+ cid: -999
+ };
textRangePage.corpusService.currentCorpus = new ReplaySubject<CorpusMC>(1);
textRangePage.corpusService.currentCorpus.next(corpus);
const validReffSpy: Spy = spyOn(textRangePage.corpusService, 'getCTSvalidReff').and.callFake(() => Promise.reject());
@@ -148,13 +153,14 @@ describe('TextRangePage', () => {
textRangePage.corpusService.initCurrentTextRange();
textRangePage.helperService.applicationState.next(textRangePage.helperService.deepCopy(MockMC.applicationState));
textRangePage.corpusService.currentCorpus = new ReplaySubject<CorpusMC>(1);
- textRangePage.corpusService.currentCorpus.next({citations: {4: new Citation()}, source_urn: ''});
+ textRangePage.corpusService.currentCorpus.next({citations: {4: new Citation()}, source_urn: '', cid: -999});
textRangePage.checkInputDisabled().then(() => {
textRangePage.isInputDisabled[0].pipe(take(1)).subscribe((isDisabled: boolean) => {
expect(isDisabled).toBe(true);
textRangePage.corpusService.currentCorpus.next({
source_urn: '',
- citations: {1: new Citation({subcitations: {2: new Citation()}})}
+ citations: {1: new Citation({subcitations: {2: new Citation()}})},
+ cid: -999
});
textRangePage.checkInputDisabled().then(() => {
textRangePage.isInputDisabled[0].pipe(take(1)).subscribe((isDisabled2: boolean) => {
@@ -179,6 +185,7 @@ describe('TextRangePage', () => {
citations: {4: new Citation()},
citation_level_2: CitationLevel.default.toString(),
citation_level_3: CitationLevel.default.toString(),
+ cid: -999
};
textRangePage.corpusService.currentCorpus = new ReplaySubject<CorpusMC>(1);
textRangePage.corpusService.currentCorpus.next(corpus);
@@ -252,7 +259,8 @@ describe('TextRangePage', () => {
textRangePage.corpusService.currentCorpus.next({
source_urn: '',
citations: {4: new Citation()},
- citation_level_2: CitationLevel.default.toString()
+ citation_level_2: CitationLevel.default.toString(),
+ cid: -999
});
textRangePage.corpusService.initCurrentTextRange();
textRangePage.helperService.applicationState.next(textRangePage.helperService.deepCopy(MockMC.applicationState));
@@ -287,7 +295,7 @@ describe('TextRangePage', () => {
textRangePage.helperService.applicationState.next(textRangePage.helperService.deepCopy(MockMC.applicationState));
textRangePage.initPage({
source_urn: '', citation_level_2: CitationLevel.default.toString(),
- citations: {1: new Citation({label: '1'})}
+ citations: {1: new Citation({label: '1'})}, cid: -999
}).then(() => {
textRangePage.corpusService.currentTextRange.pipe(take(1)).subscribe((tr: TextRange) => {
expect(tr.start[0]).toBe('1');
@@ -307,7 +315,11 @@ describe('TextRangePage', () => {
});
}
- const corpus: CorpusMC = {citations: {4: new Citation({subcitations: {}, value: 4})}, source_urn: ''};
+ const corpus: CorpusMC = {
+ citations: {4: new Citation({subcitations: {}, value: 4})},
+ source_urn: '',
+ cid: -999
+ };
textRangePage.corpusService.currentCorpus = new ReplaySubject<CorpusMC>(1);
textRangePage.corpusService.currentCorpus.next(corpus);
const valueList: number[] = [];
@@ -351,13 +363,13 @@ describe('TextRangePage', () => {
const addReferencesSpy: Spy = spyOn(textRangePage, 'addReferences').and.callFake(() => Promise.reject());
const initPageSpy: Spy = spyOn(textRangePage, 'initPage').and.returnValue(Promise.resolve());
textRangePage.corpusService.currentCorpus = new ReplaySubject<CorpusMC>(1);
- textRangePage.corpusService.currentCorpus.next({citations: {}, source_urn: ''});
+ textRangePage.corpusService.currentCorpus.next({citations: {}, source_urn: '', cid: -999});
textRangePage.ngOnInit().then(async () => {
expect(initPageSpy).toHaveBeenCalledTimes(0);
addReferencesSpy.and.returnValue(Promise.resolve());
await textRangePage.ngOnInit();
expect(initPageSpy).toHaveBeenCalledTimes(1);
- textRangePage.corpusService.currentCorpus.next({citations: {1: new Citation()}, source_urn: ''});
+ textRangePage.corpusService.currentCorpus.next({citations: {1: new Citation()}, source_urn: '', cid: -999});
await textRangePage.ngOnInit();
expect(initPageSpy).toHaveBeenCalledTimes(2);
done();
@@ -380,7 +392,7 @@ describe('TextRangePage', () => {
textRangePage.corpusService.initCurrentTextRange();
textRangePage.helperService.applicationState.next(textRangePage.helperService.deepCopy(MockMC.applicationState));
textRangePage.corpusService.currentCorpus = new ReplaySubject<CorpusMC>(1);
- textRangePage.corpusService.currentCorpus.next({citations: {1: new Citation()}, source_urn: ''});
+ textRangePage.corpusService.currentCorpus.next({citations: {1: new Citation()}, source_urn: '', cid: -999});
textRangePage.resetCitations().then(() => {
textRangePage.corpusService.currentTextRange.pipe(take(1)).subscribe((tr: TextRange) => {
expect(tr.start[1]).toBeTruthy();
@@ -414,7 +426,7 @@ describe('TextRangePage', () => {
textRangePage.helperService.applicationState.next(textRangePage.helperService.deepCopy(MockMC.applicationState));
textRangePage.corpusService.currentCorpus = new ReplaySubject<CorpusMC>(1);
textRangePage.corpusService.currentCorpus.next({
- citations: {2: new Citation({subcitations: {2: new Citation()}})}, source_urn: ''
+ citations: {2: new Citation({subcitations: {2: new Citation()}})}, source_urn: '', cid: -999
});
textRangePage.showFurtherReferences(true).then(async () => {
expect(addReferencesSpy).toHaveBeenCalledTimes(1);
@@ -424,7 +436,8 @@ describe('TextRangePage', () => {
textRangePage.corpusService.setCurrentTextRange(0, null, new TextRange({end: ['2'], start: ['2']}));
textRangePage.corpusService.currentCorpus.next({
citations: {2: new Citation({subcitations: {}})},
- citation_level_2: CitationLevel.default.toString(), source_urn: ''
+ citation_level_2: CitationLevel.default.toString(), source_urn: '',
+ cid: -999
});
await textRangePage.showFurtherReferences(false);
expect(addReferencesSpy).toHaveBeenCalledTimes(1);
diff --git a/mc_frontend/src/app/upload-text/upload-text.component.ts b/mc_frontend/src/app/upload-text/upload-text.component.ts
index 02e11db..a7b6f72 100644
--- a/mc_frontend/src/app/upload-text/upload-text.component.ts
+++ b/mc_frontend/src/app/upload-text/upload-text.component.ts
@@ -38,7 +38,7 @@ export class UploadTextComponent implements OnInit {
this.corpusService.getAnnotationsForCustomText().then((ar: AnnisResponse) => {
this.corpusService.processAnnisResponse(ar);
this.corpusService.currentUrn = ar.uri;
- const corpus: CorpusMC = {source_urn: ar.uri, title: this.corpusService.myTextString};
+ const corpus: CorpusMC = {source_urn: ar.uri, title: this.corpusService.myTextString, cid: -999};
this.corpusService.currentCorpus.next(corpus);
this.helperService.applicationState.pipe(take(1)).subscribe((state: ApplicationState) => {
state.mostRecentSetup = new TextData({
--
GitLab