From 1ee4c86e39115efd967b7e5f3dc4d897e443f893 Mon Sep 17 00:00:00 2001
From: Konstantin Schulz <schulzkx@hu-berlin.de>
Date: Tue, 27 Feb 2024 10:36:50 +0100
Subject: [PATCH] fix parameters for POST requests
---
mc_backend/app.py | 8 +--
mc_backend/mcserver/__init__.py | 3 +-
mc_backend/mcserver/__main__.py | 2 +-
mc_backend/mcserver/app/__init__.py | 52 +++++++++++++------
mc_backend/tests.py | 11 ++--
mc_frontend/src/app/corpus.service.ts | 7 +--
.../exercise-parameters.page.ts | 12 ++---
mc_frontend/src/app/exercise.service.ts | 10 ++--
mc_frontend/src/app/helper.service.spec.ts | 4 +-
mc_frontend/src/app/helper.service.ts | 15 ++++--
.../src/app/semantics/semantics.page.ts | 3 +-
.../src/app/sequences/sequences.page.ts | 9 ++--
.../src/app/show-text/show-text.page.ts | 7 +--
mc_frontend/src/app/test/test.page.ts | 5 +-
mc_frontend/src/app/vocabulary.service.ts | 3 +-
15 files changed, 79 insertions(+), 72 deletions(-)
diff --git a/mc_backend/app.py b/mc_backend/app.py
index 80aab64..031573d 100644
--- a/mc_backend/app.py
+++ b/mc_backend/app.py
@@ -1,5 +1,3 @@
-from pathlib import Path
-
import connexion
from mcserver import get_app, get_cfg
@@ -7,5 +5,7 @@ from mcserver import get_app, get_cfg
app: connexion.App = get_app()
if __name__ == "__main__":
- app.run(import_string=f"{Path(__file__).stem}:app", host=get_cfg().HOST_IP_MCSERVER, port=get_cfg().HOST_PORT,
- use_reloader=True)
+ # disable reloader (import_string) because it would result in multiple conflicting GraphANNIS instances
+ # exclude models_auto.py from watchfiles to prevent Uvicorn from running into an update/reload loop
+ app.run(host=get_cfg().HOST_IP_MCSERVER, port=get_cfg().HOST_PORT,
+ reload_excludes="models_auto.py") # import_string=f"{Path(__file__).stem}:app",
diff --git a/mc_backend/mcserver/__init__.py b/mc_backend/mcserver/__init__.py
index 53b89e2..d99c3db 100644
--- a/mc_backend/mcserver/__init__.py
+++ b/mc_backend/mcserver/__init__.py
@@ -20,5 +20,4 @@ def get_cfg() -> Type[Config]:
if __name__ == "__main__":
- # reloader has to be disabled because of a bug with Flask and multiprocessing
- get_app().run(host=get_cfg().HOST_IP_MCSERVER, port=get_cfg().HOST_PORT, use_reloader=False)
+ get_app().run(host=get_cfg().HOST_IP_MCSERVER, port=get_cfg().HOST_PORT)
diff --git a/mc_backend/mcserver/__main__.py b/mc_backend/mcserver/__main__.py
index 08ecbc7..dc21b63 100644
--- a/mc_backend/mcserver/__main__.py
+++ b/mc_backend/mcserver/__main__.py
@@ -1,4 +1,4 @@
from mcserver import get_app, get_cfg
if __name__ == "__main__":
- get_app().run(host=get_cfg().HOST_IP_MCSERVER, port=get_cfg().HOST_PORT, use_reloader=False)
+ get_app().run(host=get_cfg().HOST_IP_MCSERVER, port=get_cfg().HOST_PORT)
diff --git a/mc_backend/mcserver/app/__init__.py b/mc_backend/mcserver/app/__init__.py
index 7089358..034b3fe 100644
--- a/mc_backend/mcserver/app/__init__.py
+++ b/mc_backend/mcserver/app/__init__.py
@@ -7,7 +7,7 @@ import uuid
from logging.handlers import RotatingFileHandler
from threading import Thread
from time import strftime
-from typing import Type
+from typing import Type, List
import connexion
import open_alchemy
import prance
@@ -27,6 +27,14 @@ from starlette.middleware.cors import CORSMiddleware
from mcserver.config import Config
+class Specification:
+ def __init__(self, content: dict = None, models_last_modified_time: float = 0,
+ api_last_modified_time: float = 0):
+ self.content: dict = content
+ self.models_last_modified_time: float = models_last_modified_time
+ self.api_last_modified_time: float = api_last_modified_time
+
+
def apply_event_handlers(app: FlaskApp):
"""Applies event handlers to a given Flask application, such as logging after requests or teardown logic."""
@@ -102,29 +110,44 @@ def full_init(app: Flask, cfg: Type[Config] = Config) -> None:
start_updater(app)
-def get_api_specification() -> dict:
+def get_api_last_modified_time() -> float:
+ """Checks when the API specification was last modified."""
+ return os.path.getmtime(Config.API_SPEC_MCSERVER_FILE_PATH)
+
+
+def get_api_specification() -> Specification:
""" Reads, parses and caches the OpenAPI specification including all shared models. """
parser: prance.ResolvingParser = prance.ResolvingParser(Config.API_SPEC_MCSERVER_FILE_PATH, lazy=True, strict=False)
- api_last_modified_time: float = os.path.getmtime(Config.API_SPEC_MCSERVER_FILE_PATH)
- models_last_modified_time: float = os.path.getmtime(Config.API_SPEC_MODELS_YAML_FILE_PATH)
- lmt: float = api_last_modified_time + models_last_modified_time
+ spec: Specification = Specification(api_last_modified_time=get_api_last_modified_time(),
+ models_last_modified_time=get_models_last_modified_time())
if os.path.exists(Config.API_SPEC_CACHE_PATH):
with open(Config.API_SPEC_CACHE_PATH, "rb") as f:
- cache: dict = pickle.load(f)
- if cache["lmt"] == lmt:
- parser.specification = cache["spec"]
- if not parser.specification:
+ content: dict = pickle.load(f)
+ # check for old data to enable backward compatibility
+ if "lmt" not in content.keys():
+ cached_spec: Specification = Specification(content)
+ cached_time: List[float] = [cached_spec.api_last_modified_time, cached_spec.models_last_modified_time]
+ new_time: List[float] = [spec.api_last_modified_time, spec.models_last_modified_time]
+ if cached_time == new_time:
+ spec.content = cached_spec.content
+ if spec.content is None:
parser.parse()
+ spec.content = parser.specification
with open(Config.API_SPEC_CACHE_PATH, "wb+") as f:
- pickle.dump(dict(lmt=lmt, spec=parser.specification), f)
- return parser.specification
+ pickle.dump(spec.__dict__, f)
+ return spec
+
+
+def get_models_last_modified_time() -> float:
+ """Checks when the model specification was last modified."""
+ return os.path.getmtime(Config.API_SPEC_MODELS_YAML_FILE_PATH)
def init_app_common(cfg: Type[Config] = Config) -> connexion.App:
""" Initializes common Flask parts, e.g., CORS, configuration, database, migrations and custom corpora."""
connexion_app: connexion.FlaskApp = connexion.FlaskApp(__name__) # , specification_dir=Config.MC_SERVER_DIRECTORY
- spec: dict = get_api_specification()
- connexion_app.add_api(spec)
+ spec: Specification = get_api_specification()
+ connexion_app.add_api(spec.content)
apply_event_handlers(connexion_app)
app: Flask = connexion_app.app
# allow CORS requests for all API routes
@@ -201,8 +224,7 @@ if not hasattr(open_alchemy.models, Config.DATABASE_TABLE_CORPUS):
# initialize the database and models _BEFORE_ you add any APIs to your application
init_yaml(Config.API_SPEC_MODELS_YAML_FILE_PATH, base=db.Model,
models_filename=os.path.join(Config.MC_SERVER_DIRECTORY, "models_auto.py"))
-
-# import the models so we can access them from other parts of the app using imports from "app.models";
+# import the models, so we can access them from other parts of the app using imports from "app.models";
# this has to be at the bottom of the file
from mcserver.app import models
from mcserver.app import api
diff --git a/mc_backend/tests.py b/mc_backend/tests.py
index 3fee1da..e949e2d 100644
--- a/mc_backend/tests.py
+++ b/mc_backend/tests.py
@@ -35,7 +35,7 @@ from sqlalchemy.orm import session
import mcserver
from mcserver.app import create_app, db, start_updater, full_init, log_exception, get_api_specification, \
- init_app_common, create_postgres_database
+ init_app_common, create_postgres_database, Specification, get_models_last_modified_time, get_api_last_modified_time
from mcserver.app.api.exerciseAPI import map_exercise_data_to_database, get_graph_data
from mcserver.app.api.fileAPI import clean_tmp_folder
from mcserver.app.api.h5pAPI import get_remote_exercise
@@ -699,17 +699,16 @@ class CommonTestCase(unittest.TestCase):
def test_get_api_specification(self):
""" Reads, parses and caches the OpenAPI specification including all shared models. """
- spec: dict = get_api_specification()
+ spec: Specification = get_api_specification()
os.remove(Config.API_SPEC_CACHE_PATH)
with patch.object(mcserver.app.prance.ResolvingParser, "parse"):
get_api_specification()
self.assertTrue(os.path.exists(Config.API_SPEC_CACHE_PATH))
# restore the old cache, so that other unit tests can be initialized faster
- api_last_modified_time: float = os.path.getmtime(Config.API_SPEC_MCSERVER_FILE_PATH)
- models_last_modified_time: float = os.path.getmtime(Config.API_SPEC_MODELS_YAML_FILE_PATH)
- lmt: float = api_last_modified_time + models_last_modified_time
+ spec.api_last_modified_time = get_api_last_modified_time()
+ spec.models_last_modified_time = get_models_last_modified_time()
with open(Config.API_SPEC_CACHE_PATH, "wb+") as f:
- pickle.dump(dict(lmt=lmt, spec=spec), f)
+ pickle.dump(spec.__dict__, f)
def test_get_concept_network(self):
"""Extracts a network of words from vector data in an AI model."""
diff --git a/mc_frontend/src/app/corpus.service.ts b/mc_frontend/src/app/corpus.service.ts
index 60eb9ce..e937936 100644
--- a/mc_frontend/src/app/corpus.service.ts
+++ b/mc_frontend/src/app/corpus.service.ts
@@ -170,11 +170,8 @@ export class CorpusService {
const rtfe: RawTextFormExtension = {
plain_text: this.currentText,
};
- const formData: FormData = new FormData();
- Object.keys(rtfe).forEach((key: string) => {
- formData.append(key, rtfe[key].toString());
- });
- this.helperService.makePostRequest(this.http, this.toastCtrl, url, formData).then((ar: AnnisResponse) => {
+ const body: HttpParams = this.helperService.getFormForPostRequest(rtfe);
+ this.helperService.makePostRequest(this.http, this.toastCtrl, url, body).then((ar: AnnisResponse) => {
return resolve(ar);
}, (error: HttpErrorResponse) => {
return reject(error);
diff --git a/mc_frontend/src/app/exercise-parameters/exercise-parameters.page.ts b/mc_frontend/src/app/exercise-parameters/exercise-parameters.page.ts
index df90bae..32973ab 100644
--- a/mc_frontend/src/app/exercise-parameters/exercise-parameters.page.ts
+++ b/mc_frontend/src/app/exercise-parameters/exercise-parameters.page.ts
@@ -6,7 +6,7 @@ import {
PhenomenonTranslation
} from '../models/enum';
import {NavController, ToastController} from '@ionic/angular';
-import {HttpClient} from '@angular/common/http';
+import {HttpClient, HttpParams} from '@angular/common/http';
import {Component, OnInit} from '@angular/core';
import {TranslateService} from '@ngx-translate/core';
import {ExerciseService} from 'src/app/exercise.service';
@@ -132,10 +132,7 @@ export class ExerciseParametersPage implements OnInit {
getH5Pexercise(ef: ExerciseForm): Promise<void> {
return new Promise<void>((resolve, reject) => {
const url: string = configMC.backendBaseUrl + configMC.backendApiExercisePath;
- const formData = new FormData();
- Object.keys(ef).forEach((key: string) => {
- formData.append(key, ef[key]);
- });
+ const formData: HttpParams = this.helperService.getFormForPostRequest(ef);
this.helperService.makePostRequest(this.http, this.toastCtrl, url, formData).then((ar: AnnisResponse) => {
this.helperService.applicationState.pipe(take(1)).subscribe((as: ApplicationState) => {
as.previewAnnisResponse = this.corpusService.previewAnnisResponse = ar;
@@ -157,11 +154,8 @@ export class ExerciseParametersPage implements OnInit {
search_values: searchValues,
urn: this.corpusService.currentUrn
};
- const formData = new FormData();
- Object.keys(kf).forEach((key: string) => {
- formData.append(key, kf[key]);
- });
const kwicUrl: string = configMC.backendBaseUrl + configMC.backendApiKwicPath;
+ const formData: HttpParams = this.helperService.getFormForPostRequest(kf);
this.helperService.makePostRequest(this.http, this.toastCtrl, kwicUrl, formData).then((svgString: string) => {
this.exerciseService.kwicGraphs = svgString;
this.helperService.goToPage(this.navCtrl, configMC.pageUrlKwic).then();
diff --git a/mc_frontend/src/app/exercise.service.ts b/mc_frontend/src/app/exercise.service.ts
index 4a944e1..cf384dd 100644
--- a/mc_frontend/src/app/exercise.service.ts
+++ b/mc_frontend/src/app/exercise.service.ts
@@ -137,12 +137,9 @@ export class ExerciseService {
lang: this.currentExerciseParams.language.toString(),
solution_indices: indices
};
- const formData: FormData = new FormData();
- Object.keys(h5pForm).forEach((key: string) => {
- formData.append(key, h5pForm[key].toString());
- });
- const options = {responseType: 'blob' as const};
+ const options: object = {responseType: 'blob' as const};
const errorMsg: string = HelperService.generalErrorAlertMessage;
+ const formData: HttpParams = this.helperService.getFormForPostRequest(h5pForm);
this.helperService.makePostRequest(this.http, this.toastCtrl, url, formData, errorMsg, options)
.then((result: Blob) => {
const fileName = this.currentExerciseParams.type + '.h5p';
@@ -338,8 +335,7 @@ export class ExerciseService {
sendData(result: TestResultMC): Promise<void> {
return new Promise<void>((resolve, reject) => {
const fileUrl: string = configMC.backendBaseUrl + configMC.backendApiFilePath;
- const formData = new FormData();
- formData.append('learning_result', JSON.stringify({0: result.statement}));
+ const formData: HttpParams = new HttpParams().set('learning_result', JSON.stringify({0: result.statement}));
this.helperService.makePostRequest(this.http, this.toastCtrl, fileUrl, formData, '').then(() => {
return resolve();
}, () => {
diff --git a/mc_frontend/src/app/helper.service.spec.ts b/mc_frontend/src/app/helper.service.spec.ts
index e9fb48b..90fe76d 100644
--- a/mc_frontend/src/app/helper.service.spec.ts
+++ b/mc_frontend/src/app/helper.service.spec.ts
@@ -209,11 +209,11 @@ describe('HelperService', () => {
const toastCtrl: ToastController = TestBed.inject(ToastController);
spyOn(toastCtrl, 'create').and.returnValue(Promise.resolve({present: () => Promise.resolve()} as HTMLIonToastElement));
const httpSpy: Spy = spyOn(helperService.http, 'post').and.returnValue(of(0));
- helperService.makePostRequest(helperService.http, toastCtrl, '', new FormData()).then((result: number) => {
+ helperService.makePostRequest(helperService.http, toastCtrl, '', new HttpParams()).then((result: number) => {
expect(httpSpy).toHaveBeenCalledTimes(1);
expect(result).toBe(0);
httpSpy.and.returnValue(new Observable(subscriber => subscriber.error(new HttpErrorResponse({status: 500}))));
- helperService.makePostRequest(helperService.http, toastCtrl, '', new FormData()).then(() => {
+ helperService.makePostRequest(helperService.http, toastCtrl, '', new HttpParams()).then(() => {
}, (error: HttpErrorResponse) => {
expect(error.status).toBe(500);
done();
diff --git a/mc_frontend/src/app/helper.service.ts b/mc_frontend/src/app/helper.service.ts
index 16e666b..0cd656a 100644
--- a/mc_frontend/src/app/helper.service.ts
+++ b/mc_frontend/src/app/helper.service.ts
@@ -1,5 +1,5 @@
/* tslint:disable:no-string-literal */
-import {HttpClient, HttpErrorResponse, HttpParams} from '@angular/common/http';
+import {HttpClient, HttpErrorResponse, HttpHeaders, HttpParams} from '@angular/common/http';
import {Injectable} from '@angular/core';
import {NavController, ToastController} from '@ionic/angular';
import {ApplicationState} from 'src/app/models/applicationState';
@@ -207,6 +207,14 @@ export class HelperService {
});
}
+ getFormForPostRequest(form: object): HttpParams {
+ let params: HttpParams = new HttpParams();
+ Object.keys(form).forEach((key: string) => {
+ params = params.set(key, form[key]);
+ });
+ return params;
+ }
+
getH5P(): any {
return H5P;
}
@@ -324,12 +332,13 @@ export class HelperService {
}));
}
- makePostRequest(http: HttpClient, toastCtrl: ToastController, url: string, formData: FormData,
+ makePostRequest(http: HttpClient, toastCtrl: ToastController, url: string, body: HttpParams,
errorMessage: string = HelperService.generalErrorAlertMessage, options: any = {}): Promise<any> {
return new Promise(((resolve, reject) => {
this.currentError = null;
this.openRequests.push(url);
- http.post(url, formData, options).subscribe((result: any) => {
+ options.headers = new HttpHeaders().set('Content-Type', 'application/x-www-form-urlencoded');
+ http.post(url, body, options).subscribe((result: any) => {
this.openRequests.splice(this.openRequests.indexOf(url), 1);
return resolve(result);
}, async (error: HttpErrorResponse) => {
diff --git a/mc_frontend/src/app/semantics/semantics.page.ts b/mc_frontend/src/app/semantics/semantics.page.ts
index 98f13e3..bf6e74b 100644
--- a/mc_frontend/src/app/semantics/semantics.page.ts
+++ b/mc_frontend/src/app/semantics/semantics.page.ts
@@ -43,8 +43,7 @@ export class SemanticsPage implements AfterViewInit {
search_regex: this.searchRegex,
nearest_neighbor_count: Math.max(Math.round(this.nearestNeighborCount), 1)
};
- const formData: FormData = new FormData();
- Object.keys(vnf).forEach((key: string) => formData.append(key, vnf[key]));
+ const formData: HttpParams = this.helperService.getFormForPostRequest(vnf);
const semanticsUrl: string = configMC.backendBaseUrl + configMC.backendApiVectorNetworkPath;
this.similarContexts = [];
this.helperService.makePostRequest(this.http, this.toastCtrl, semanticsUrl, formData).then((contexts: string[][]) => {
diff --git a/mc_frontend/src/app/sequences/sequences.page.ts b/mc_frontend/src/app/sequences/sequences.page.ts
index e283b72..045b583 100644
--- a/mc_frontend/src/app/sequences/sequences.page.ts
+++ b/mc_frontend/src/app/sequences/sequences.page.ts
@@ -4,7 +4,7 @@ import {NavController, ToastController} from '@ionic/angular';
import configMC from '../../configMC';
import {ExerciseTypePath, ZenodoRecord} from '../../../openapi';
import {ExerciseService} from '../exercise.service';
-import {HttpClient} from '@angular/common/http';
+import {HttpClient, HttpParams} from '@angular/common/http';
import {ZenodoRecordMC} from '../models/zenodoRecordMC';
import {ExerciseParams} from '../models/exerciseParams';
import {TranslateService} from '@ngx-translate/core';
@@ -86,11 +86,8 @@ export class SequencesPage implements OnInit {
} else {
const url: string = configMC.backendBaseUrl + configMC.backendApiZenodoPath;
const zenodoForm: ZenodoForm = {record_id: +zr.record.identifier[0].split('.').slice(-1)[0]};
- const formData: FormData = new FormData();
- Object.keys(zenodoForm).forEach((key: string) => {
- formData.append(key, zenodoForm[key]);
- });
- this.helperService.makePostRequest(this.http, this.toastCtrl, url, formData).then((uris: string[]) => {
+ const params: HttpParams = this.helperService.getFormForPostRequest(zenodoForm);
+ this.helperService.makePostRequest(this.http, this.toastCtrl, url, params).then((uris: string[]) => {
zr.fileURIs = uris;
zr.showFiles = !zr.showFiles;
return resolve();
diff --git a/mc_frontend/src/app/show-text/show-text.page.ts b/mc_frontend/src/app/show-text/show-text.page.ts
index 12f4da3..9cb4ab5 100644
--- a/mc_frontend/src/app/show-text/show-text.page.ts
+++ b/mc_frontend/src/app/show-text/show-text.page.ts
@@ -6,7 +6,7 @@ import {VocabularyService} from 'src/app/vocabulary.service';
import {ExerciseService} from 'src/app/exercise.service';
import {HelperService} from 'src/app/helper.service';
import {TranslateService} from '@ngx-translate/core';
-import {HttpClient} from '@angular/common/http';
+import {HttpClient, HttpParams} from '@angular/common/http';
import {CorpusMC} from '../models/corpusMC';
import {take} from 'rxjs/operators';
import configMC from '../../configMC';
@@ -64,13 +64,10 @@ export class ShowTextPage implements OnInit {
this.corpusService.currentCorpus.pipe(take(1)).subscribe((cc: CorpusMC) => {
const authorTitle: string = cc.author + ', ' + cc.title;
content = `<p>${authorTitle} ${this.corpusService.currentUrn.split(':').slice(-1)[0]}</p>` + content;
- const formData = new FormData();
const ff: FileForm = {
file_type: fileType as FileType, html_content: content, urn: this.corpusService.currentUrn
};
- Object.keys(ff).forEach((key: string) => {
- formData.append(key, ff[key]);
- });
+ let formData: HttpParams = this.helperService.getFormForPostRequest(ff);
const url: string = configMC.backendBaseUrl + configMC.backendApiFilePath;
this.isDownloading = true;
this.helperService.makePostRequest(this.http, this.toastCtrl, url, formData)
diff --git a/mc_frontend/src/app/test/test.page.ts b/mc_frontend/src/app/test/test.page.ts
index ec013f8..e07042c 100644
--- a/mc_frontend/src/app/test/test.page.ts
+++ b/mc_frontend/src/app/test/test.page.ts
@@ -10,7 +10,7 @@ import {ConfirmCancelPage} from 'src/app/confirm-cancel/confirm-cancel.page';
import {ExercisePart} from 'src/app/models/exercisePart';
import Activity from 'src/app/models/xAPI/Activity';
import LanguageMap from 'src/app/models/xAPI/LanguageMap';
-import {HttpClient} from '@angular/common/http';
+import {HttpClient, HttpParams} from '@angular/common/http';
import Context from 'src/app/models/xAPI/Context';
import {TestResultMC} from 'src/app/models/testResultMC';
import {ExerciseService} from 'src/app/exercise.service';
@@ -380,12 +380,11 @@ export class TestPage implements OnDestroy, OnInit {
return resolve();
}
const fileUrl: string = configMC.backendBaseUrl + configMC.backendApiFilePath;
- const formData = new FormData();
// tslint:disable-next-line:prefer-const
let learningResult: object = {};
Object.keys(this.vocService.currentTestResults)
.forEach(i => learningResult[i] = this.vocService.currentTestResults[i].statement);
- formData.append('learning_result', JSON.stringify(learningResult));
+ const formData: HttpParams = new HttpParams().set('learning_result', JSON.stringify(learningResult));
this.helperService.makePostRequest(this.http, this.toastCtrl, fileUrl, formData).then(async () => {
this.wasDataSent = true;
this.helperService.showToast(this.toastCtrl, this.corpusService.dataSentSuccessMessage).then();
diff --git a/mc_frontend/src/app/vocabulary.service.ts b/mc_frontend/src/app/vocabulary.service.ts
index c995037..97d6888 100644
--- a/mc_frontend/src/app/vocabulary.service.ts
+++ b/mc_frontend/src/app/vocabulary.service.ts
@@ -103,8 +103,7 @@ export class VocabularyService implements OnInit {
query_urn: queryUrn,
vocabulary: this.currentReferenceVocabulary
};
- const formData: FormData = new FormData();
- Object.keys(vf).forEach((key: string) => formData.append(key, vf[key]));
+ let formData: HttpParams = this.helperService.getFormForPostRequest(vf);
this.helperService.makePostRequest(this.http, this.toastCtrl, url, formData).then((result: AnnisResponse) => {
return resolve(result);
}, (error: HttpErrorResponse) => {
--
GitLab