fixed graph database creation in docker environments and persistence of...

fixed graph database creation in docker environments and persistence of vocabulary check information in the frontend

.gitlab-ci.yml

@@ -5,8 +5,6 @@ stages:
 build:
   stage: build
   script:
-    - pwd
-    - ls
     - python3 ./mc_frontend/update_version.py
     - docker-compose build
 ci_frontend:

deploy.sh

-python3 update_version.py
+python3 ./mc_frontend/update_version.py
 docker-compose build
 docker-compose down
 docker-compose up -d

mc_backend/mcserver/app/__init__.py

@@ -2,6 +2,7 @@
 import logging
 import os
 import sys
+import uuid
 from logging.handlers import RotatingFileHandler
 from threading import Thread
 from time import strftime
@@ -47,6 +48,7 @@ def create_app(cfg: Type[Config] = Config) -> Flask:
     # use local postgres database for migrations
     if len(sys.argv) > 2 and sys.argv[2] == Config.FLASK_MIGRATE:
         cfg.SQLALCHEMY_DATABASE_URI = Config.DATABASE_URL_LOCAL
+    Config.GRAPH_DATABASE_DIR = os.path.join(Config.GRAPH_DATABASE_DIR_BASE, str(uuid.uuid4()))
     Config.CORPUS_STORAGE_MANAGER = CorpusStorageManager(Config.GRAPH_DATABASE_DIR)
     app: Flask = init_app_common(cfg=cfg)
     from mcserver.app.services import bp as services_bp

mc_backend/mcserver/app/api/h5pAPI.py

@@ -7,7 +7,6 @@ import connexion
 from connexion.lifecycle import ConnexionResponse
 from flask import Response, send_from_directory
 from mcserver import Config
-from mcserver.app import db
 from mcserver.app.models import Language, ExerciseType, Solution, MimeType, FileType
 from mcserver.app.services import TextService, NetworkService, DatabaseService
 from mcserver.models_auto import Exercise

mc_backend/mcserver/config.py

@@ -19,7 +19,8 @@ class Config(object):
     CURRENT_WORKING_DIRECTORY = os.getcwd()
     CURRENT_WORKING_DIRECTORY_PARENT = os.path.dirname(CURRENT_WORKING_DIRECTORY)
     CURRENT_WORKING_DIRECTORY_PARTS = os.path.split(CURRENT_WORKING_DIRECTORY)  # [::-1]
-    GRAPH_DATABASE_DIR = os.path.join(os.sep, "tmp", "graphannis-data", str(os.getpid()))
+    GRAPH_DATABASE_DIR_BASE = os.path.join(os.sep, "tmp", "graphannis-data")
+    GRAPH_DATABASE_DIR = GRAPH_DATABASE_DIR_BASE
     MC_SERVER_DIRECTORY = CURRENT_WORKING_DIRECTORY if \
         os.path.split(CURRENT_WORKING_DIRECTORY)[-1] == "mcserver" else os.path.join(CURRENT_WORKING_DIRECTORY,
                                                                                      "mcserver")

mc_backend/tests.py

@@ -855,7 +855,7 @@ class CommonTestCase(unittest.TestCase):
         CorpusService.get_corpus(Mocks.urn_custom)
         with patch.object(CorpusService, "get_corpus", return_value=Mocks.annis_response):
             fa = CorpusService.get_frequency_analysis(Mocks.urn_custom)
-            self.assertEqual(len(fa), 163)
+            self.assertEqual(len(fa), 214)
 
     def test_get_graph(self):
         """ Retrieves a graph from the cache or, if not there, builds it from scratch. """
@@ -1197,7 +1197,7 @@ if __name__ == '__main__':
     suite.addTests(TestLoader().loadTestsFromTestCase(CorpusTestCase))
     suite.addTests(TestLoader().loadTestsFromTestCase(CommonTestCase))
     runner.run(suite)
-    if os.path.exists(Config.GRAPH_DATABASE_DIR):
-        shutil.rmtree(Config.GRAPH_DATABASE_DIR)
+    if os.path.exists(Config.GRAPH_DATABASE_DIR_BASE):
+        shutil.rmtree(Config.GRAPH_DATABASE_DIR_BASE)
     # delete the SQLITE database to have a clean start next time
     os.remove(TestingConfig.SQLALCHEMY_DATABASE_URI[10:])

mc_frontend/src/app/corpus.service.ts

@@ -127,6 +127,9 @@ export class CorpusService {
             this.helperService.applicationState.pipe(take(1)).subscribe((state: ApplicationState) => {
                 this.previewAnnisResponse = state.previewAnnisResponse;
                 this.annisResponse = state.mostRecentSetup.annisResponse;
+                this.helperService.isVocabularyCheck = [this.previewAnnisResponse, this.annisResponse].some(
+                    (ar: AnnisResponse) => ar && ar.graph_data && ar.graph_data.nodes && ar.graph_data.nodes.some(
+                        (x: NodeMC) => x.is_oov));
                 this.currentAuthor = state.mostRecentSetup.currentAuthor;
                 this.currentUrn = state.mostRecentSetup.currentUrn;
                 this.currentCorpusCache = state.mostRecentSetup.currentCorpus;
@@ -513,6 +516,7 @@ export class CorpusService {
                 this.currentUrn = state.mostRecentSetup.currentUrn;
                 this.currentCorpusCache = state.mostRecentSetup.currentCorpus;
                 this.currentTextRangeCache = state.mostRecentSetup.currentTextRange;
+                this.previewAnnisResponse = state.previewAnnisResponse;
                 this.isTextRangeCorrect = true;
                 if (this.annisResponse && this.annisResponse.graph_data && this.annisResponse.graph_data.nodes.length) {
                     this.processAnnisResponse(this.annisResponse, false);

mc_frontend/src/app/exercise-parameters/exercise-parameters.page.ts

@@ -139,9 +139,6 @@ export class ExerciseParametersPage implements OnInit {
                 this.helperService.applicationState.pipe(take(1)).subscribe((as: ApplicationState) => {
                     as.previewAnnisResponse = this.corpusService.previewAnnisResponse = ar;
                     this.helperService.saveApplicationState(as).then();
-                    // this.corpusService.annisResponse.exercise_id = ar.exercise_id;
-                    // this.corpusService.annisResponse.uri = ar.uri;
-                    // this.corpusService.annisResponse.solutions = ar.solutions;
                     this.helperService.goToPage(this.navCtrl, configMC.pageUrlPreview).then();
                     return resolve();
                 });

mc_frontend/src/app/exercise.service.ts

@@ -5,7 +5,7 @@ import {HelperService} from './helper.service';
 import {ExercisePart} from './models/exercisePart';
 import {EventMC, ExerciseType, MoodleExerciseType} from './models/enum';
 import {HttpClient, HttpParams} from '@angular/common/http';
-import {AnnisResponse, ExerciseTypePath, H5PForm, Solution} from '../../openapi';
+import {AnnisResponse, ExerciseTypePath, H5PForm, Solution, SolutionElement} from '../../openapi';
 import {take} from 'rxjs/operators';
 import {ApplicationState} from './models/applicationState';
 import {ToastController} from '@ionic/angular';
@@ -79,6 +79,10 @@ export class ExerciseService {
                 public translateService: TranslateService) {
     }
 
+    areSolutionElementsEqual(se1: SolutionElement, se2: SolutionElement): boolean {
+        return se1 === se2 ? true : se1 && se2 && se1.salt_id === se2.salt_id;
+    }
+
     createGuid(): string {
         function s4(): string {
             return Math.floor((1 + Math.random()) * 0x10000)
@@ -141,10 +145,6 @@ export class ExerciseService {
         });
     }
 
-    getJSzip(): JSZip {
-        return new JSZip();
-    }
-
     downloadLocalH5PExercise(): Promise<void> {
         return new Promise<void>((resolve, reject) => {
             this.helperService.makeGetRequest(this.http, this.toastCtrl, this.currentExerciseParams.filePath)
@@ -191,9 +191,16 @@ export class ExerciseService {
         return document.querySelector(this.h5pIframeString);
     }
 
+    getJSzip(): JSZip {
+        return new JSZip();
+    }
+
     getSolutionIndices(solutions: Solution[]): string {
         const indices: string[] =
-            solutions.map(x => this.corpusService.previewAnnisResponse.solutions.indexOf(x).toString());
+            solutions.map((x: Solution) => this.corpusService.previewAnnisResponse.solutions.findIndex((y: Solution) => {
+                return this.areSolutionElementsEqual(x.target, y.target) &&
+                    this.areSolutionElementsEqual(x.value, y.value);
+            }).toString());
         return '&solution_indices=' + indices.join(',');
     }
 

mc_frontend/src/app/preview/preview.page.spec.ts

@@ -146,7 +146,7 @@ describe('PreviewPage', () => {
             }];
         previewPage.corpusService.exercise.type = ExerciseType.markWords;
         previewPage.exerciseService.excludeOOV = true;
-        previewPage.corpusService.previewAnnisResponse = {
+        previewPage.corpusService.previewAnnisResponse = previewPage.corpusService.annisResponse = {
             graph_data: {nodes: [{is_oov: false, id: 'id'}], links: []},
             solutions
         };

mc_frontend/src/app/preview/preview.page.ts

@@ -79,7 +79,7 @@ export class PreviewPage implements AfterContentInit, OnDestroy, OnInit {
     processSolutions(solutions: Solution[]): void {
         const isCloze: boolean = this.corpusService.exercise.type === ExerciseType.cloze;
         if (this.exerciseService.excludeOOV) {
-            const nodeIdSet: Set<string> = new Set(this.corpusService.previewAnnisResponse.graph_data.nodes.filter(
+            const nodeIdSet: Set<string> = new Set(this.corpusService.annisResponse.graph_data.nodes.filter(
                 x => !x.is_oov).map(x => x.id));
             solutions = this.corpusService.previewAnnisResponse.solutions.filter(
                 x => nodeIdSet.has(x.target.salt_id) && (isCloze || nodeIdSet.has(x.value.salt_id)));
@@ -88,9 +88,9 @@ export class PreviewPage implements AfterContentInit, OnDestroy, OnInit {
         if (isCloze) {
             this.maxGapLength = Math.max.apply(Math, solutions.map(x => x.target.content.length));
             this.solutionNodeIdSet = new Set(solutions.map(x => x.target.salt_id));
-            newSolutions = solutions.concat();
+            newSolutions = solutions;
         } else {
-            newSolutions = solutions.concat().sort((s1, s2) => {
+            newSolutions = solutions.sort((s1, s2) => {
                 return s1.target.content < s2.target.content ? -1 : (s1.target.content > s2.target.content ? 1 : 0);
             });
         }