added the Zenodo API for retrieving records with exercise materials from specific communities

docker-compose.yml

@@ -2,7 +2,7 @@ version: '3.7'
 
 services:
   db:
-    image: postgres
+    image: postgres:12-alpine
     environment:
       - POSTGRES_HOST_AUTH_METHOD=trust
 #    ports:

mc_backend/mcserver/__main__.py

 from mcserver import get_app, get_cfg
 
-get_app().run(host=get_cfg().HOST_IP_MCSERVER, port=get_cfg().HOST_PORT, use_reloader=False)
+if __name__ == "__main__":
+    get_app().run(host=get_cfg().HOST_IP_MCSERVER, port=get_cfg().HOST_PORT, use_reloader=False)

mc_backend/mcserver/app/api/__init__.py

@@ -6,4 +6,4 @@ bp = Blueprint("api", __name__)
 api = Api(bp)
 
 from . import corpusAPI, corpusListAPI, exerciseAPI, exerciseListAPI, fileAPI, frequencyAPI, h5pAPI, kwicAPI, \
-    rawTextAPI, staticExercisesAPI, textcomplexityAPI, validReffAPI, vectorNetworkAPI, vocabularyAPI
+    rawTextAPI, staticExercisesAPI, textcomplexityAPI, validReffAPI, vectorNetworkAPI, vocabularyAPI, zenodoAPI

mc_backend/mcserver/app/api/zenodoAPI.py

+"""The Zenodo API. Add it to your REST API to provide users with exercise materials from the Zenodo repository."""
+from sickle import Sickle
+from typing import List, Set, Any, Dict
+import inspect
+from sickle.models import Record
+
+from mcserver import Config
+from mcserver.app.services import NetworkService
+from openapi.openapi_server.models import ZenodoRecord
+
+
+def get():
+    """The GET method for the Zenodo REST API. It provides exercise materials from the Zenodo repository."""
+    record_members: List[tuple] = inspect.getmembers(ZenodoRecord, lambda a: not (inspect.isroutine(a)))
+    record_properties: Set[str] = set([x[0] for x in record_members if type(x[1]) == property])
+    author_property: str = "Autor:"
+    work_property: str = "Werk:"
+    sickle: Sickle = Sickle(Config.ZENODO_API_URL)
+    records: List[Record] = list(sickle.ListRecords(metadataPrefix='oai_dc', set=Config.ZENODO_SET))
+    zenodo_records: List[ZenodoRecord] = []
+    for record in records:
+        md: dict = record.metadata
+        params: Dict[str, Any] = {rp: md.get(rp, [None])[0] for rp in record_properties}
+        new_record: ZenodoRecord = ZenodoRecord(**params)
+        tags: List[str] = md.get("subject", [])
+        new_record.author = next((x.split(author_property)[-1] for x in tags if x.startswith(author_property)), "")
+        new_record.work = next((x.split(work_property)[-1] for x in tags if x.startswith(work_property)), "")
+        zenodo_records.append(new_record)
+    return NetworkService.make_json_response([x.to_dict() for x in zenodo_records])

mc_backend/mcserver/app/services/corpusService.py

@@ -372,7 +372,7 @@ class CorpusService:
     @staticmethod
     def update_corpora():
         """Checks the remote repositories for new corpora to be included in our database."""
-        CorpusService.existing_corpora = db.session.query(Corpus).all()
+        CorpusService.existing_corpora = DatabaseService.query(Corpus)
         resolver: HttpCtsRetriever = HttpCtsRetriever(Config.CTS_API_BASE_URL)
         # check the appropriate literature for the desired author
         resp: str = resolver.getCapabilities(urn="urn:cts:latinLit")  # "urn:cts:greekLit" for Greek
@@ -400,14 +400,15 @@ class CorpusService:
                 if urn not in urn_set_existing:
                     CorpusService.add_corpus(title_value, urn, group_name_value, citation_levels)
                 else:
-                    corpus_to_update: Corpus = db.session.query(Corpus).filter_by(source_urn=urn).first()
-                    CorpusService.update_corpus(title_value, urn, group_name_value, citation_levels, corpus_to_update)
+                    corpus_to_update: Corpus = DatabaseService.query(Corpus, dict(source_urn=urn), True)
+                    CorpusService.update_corpus(
+                        title_value, urn, group_name_value, citation_levels, corpus_to_update)
         for urn in urn_set_existing:
             if urn not in urn_set_new:
-                corpus_to_delete: Corpus = db.session.query(Corpus).filter_by(source_urn=urn).first()
+                corpus_to_delete: Corpus = DatabaseService.query(Corpus, dict(source_urn=urn), True)
                 db.session.delete(corpus_to_delete)
                 db.session.commit()
-        CorpusService.existing_corpora = db.session.query(Corpus).all()
+        CorpusService.existing_corpora = DatabaseService.query(Corpus)
         db.session.commit()
 
     @staticmethod

mc_backend/mcserver/config.py

@@ -137,6 +137,7 @@ class Config(object):
     SERVER_URI_VALID_REFF = SERVER_URI_BASE + "validReff"
     SERVER_URI_VECTOR_NETWORK = SERVER_URI_BASE + "vectorNetwork"
     SERVER_URI_VOCABULARY = SERVER_URI_BASE + "vocabulary"
+    SERVER_URI_ZENODO = SERVER_URI_BASE + "zenodo"
     # END endpoints
     SQLALCHEMY_DATABASE_URI = os.environ.get("DATABASE_URL") or DATABASE_URL_SQLITE
     # BEWARE: if True, logs every single database statement executed by this application to STDOUT
@@ -161,6 +162,8 @@ class Config(object):
     VOCABULARY_PROIEL_FILE_NAME = "vocabulary_proiel_treebank.json"
     VOCABULARY_VISCHER_FILE_NAME = "vocabulary_vischer.json"
     VOCABULARY_VIVA_FILE_NAME = "vocabulary_viva.json"
+    ZENODO_API_URL = "https://zenodo.org/oai2d"
+    ZENODO_SET = "user-lexiprojekt-latin-unipotsdam"
 
 
 class ProductionConfig(Config):

mc_backend/mcserver/mcserver_api.yaml

@@ -19,13 +19,7 @@ paths:
               schema:
                 $ref: '../openapi_models.yaml#/components/schemas/Corpus'
       parameters:
-        - name: last_update_time
-          in: query
-          description: Time (in milliseconds) of the last update.
-          required: true
-          schema:
-            type: integer
-            example: 123456789
+        - $ref: '../openapi_models.yaml#/components/parameters/LastUpdateTimeParam'
   /corpora/{cid}:
     parameters:
       - name: cid
@@ -232,7 +226,7 @@ paths:
               schema:
                 type: object
                 description: JSON template for an interactive H5P exercise.
-                example: {}
+                example: { }
       parameters:
         - $ref: '../openapi_models.yaml#/components/parameters/EidParam'
         - $ref: '../openapi_models.yaml#/components/parameters/LangParam'
@@ -386,7 +380,7 @@ paths:
                 items:
                   type: array
                   description: Tokenized sentence.
-                  example: [in, vino, veritas]
+                  example: [ in, vino, veritas ]
                   items:
                     type: string
                     description: Token of a sentence.
@@ -447,6 +441,19 @@ paths:
           application/x-www-form-urlencoded:
             schema:
               $ref: '../openapi_models.yaml#/components/schemas/VocabularyForm'
+  /zenodo:
+    get:
+      summary: Shows which exercises are available on Zenodo.
+      operationId: mcserver.app.api.zenodoAPI.get
+      responses:
+        "200":
+          description: Retrieves records from Zenodo communities.
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: '../openapi_models.yaml#/components/schemas/ZenodoRecord'
 # include this here so the data model gets generated correctly
 components:
   schemas:

mc_backend/openapi/openapi_server/controllers/default_controller.py

@@ -11,6 +11,7 @@ from openapi.openapi_server.models.sentence import Sentence  # noqa: E501
 from openapi.openapi_server.models.static_exercise import StaticExercise  # noqa: E501
 from openapi.openapi_server.models.text_complexity import TextComplexity  # noqa: E501
 from openapi.openapi_server.models.vocabulary_mc import VocabularyMC  # noqa: E501
+from openapi.openapi_server.models.zenodo_record import ZenodoRecord  # noqa: E501
 from openapi.openapi_server import util
 
 
@@ -349,3 +350,16 @@ def mcserver_app_api_vocabulary_api_post(frequency_upper_bound, query_urn, vocab
     if connexion.request.is_json:
         vocabulary = VocabularyMC.from_dict(connexion.request.get_json())  # noqa: E501
     return 'do some magic!'
+
+
+def mcserver_app_api_zenodo_api_get(last_update_time):  # noqa: E501
+    """Shows which exercises are available on Zenodo.
+
+     # noqa: E501
+
+    :param last_update_time: Time (in milliseconds) of the last update.
+    :type last_update_time: int
+
+    :rtype: List[ZenodoRecord]
+    """
+    return 'do some magic!'

mc_backend/openapi/openapi_server/models/__init__.py

@@ -28,8 +28,7 @@ from openapi.openapi_server.models.solution import Solution
 from openapi.openapi_server.models.solution_element import SolutionElement
 from openapi.openapi_server.models.static_exercise import StaticExercise
 from openapi.openapi_server.models.text_complexity import TextComplexity
-from openapi.openapi_server.models.text_complexity_form import TextComplexityForm
-from openapi.openapi_server.models.text_complexity_form_extension import TextComplexityFormExtension
 from openapi.openapi_server.models.vector_network_form import VectorNetworkForm
 from openapi.openapi_server.models.vocabulary_form import VocabularyForm
 from openapi.openapi_server.models.vocabulary_mc import VocabularyMC
+from openapi.openapi_server.models.zenodo_record import ZenodoRecord

mc_backend/openapi/openapi_server/models/record.py

+# coding: utf-8
+
+from __future__ import absolute_import
+from datetime import date, datetime  # noqa: F401
+
+from typing import List, Dict  # noqa: F401
+
+from openapi.openapi_server.models.base_model_ import Model
+from openapi.openapi_server import util
+
+
+class Record(Model):
+    """NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
+
+    Do not edit the class manually.
+    """
+
+    def __init__(self, description=None, title=None):  # noqa: E501
+        """Record - a model defined in OpenAPI
+
+        :param description: The description of this Record.  # noqa: E501
+        :type description: str
+        :param title: The title of this Record.  # noqa: E501
+        :type title: str
+        """
+        self.openapi_types = {
+            'description': str,
+            'title': str
+        }
+
+        self.attribute_map = {
+            'description': 'description',
+            'title': 'title'
+        }
+
+        self._description = description
+        self._title = title
+
+    @classmethod
+    def from_dict(cls, dikt) -> 'Record':
+        """Returns the dict as a model
+
+        :param dikt: A dict.
+        :type: dict
+        :return: The Record of this Record.  # noqa: E501
+        :rtype: Record
+        """
+        return util.deserialize_model(dikt, cls)
+
+    @property
+    def description(self):
+        """Gets the description of this Record.
+
+        Description of the record.  # noqa: E501
+
+        :return: The description of this Record.
+        :rtype: str
+        """
+        return self._description
+
+    @description.setter
+    def description(self, description):
+        """Sets the description of this Record.
+
+        Description of the record.  # noqa: E501
+
+        :param description: The description of this Record.
+        :type description: str
+        """
+
+        self._description = description
+
+    @property
+    def title(self):
+        """Gets the title of this Record.
+
+        Title of the record.  # noqa: E501
+
+        :return: The title of this Record.
+        :rtype: str
+        """
+        return self._title
+
+    @title.setter
+    def title(self, title):
+        """Sets the title of this Record.
+
+        Title of the record.  # noqa: E501
+
+        :param title: The title of this Record.
+        :type title: str
+        """
+
+        self._title = title

mc_backend/openapi/openapi_server/models/zenodo_record.py

+# coding: utf-8
+
+from __future__ import absolute_import
+from datetime import date, datetime  # noqa: F401
+
+from typing import List, Dict  # noqa: F401
+
+from openapi.openapi_server.models.base_model_ import Model
+from openapi.openapi_server import util
+
+
+class ZenodoRecord(Model):
+    """NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
+
+    Do not edit the class manually.
+    """
+
+    def __init__(self, author=None, contributor=None, creator=None, description=None, identifier=None, language=None, subject=None, title=None, work=None):  # noqa: E501
+        """ZenodoRecord - a model defined in OpenAPI
+
+        :param author: The author of this ZenodoRecord.  # noqa: E501
+        :type author: str
+        :param contributor: The contributor of this ZenodoRecord.  # noqa: E501
+        :type contributor: str
+        :param creator: The creator of this ZenodoRecord.  # noqa: E501
+        :type creator: str
+        :param description: The description of this ZenodoRecord.  # noqa: E501
+        :type description: str
+        :param identifier: The identifier of this ZenodoRecord.  # noqa: E501
+        :type identifier: str
+        :param language: The language of this ZenodoRecord.  # noqa: E501
+        :type language: str
+        :param subject: The subject of this ZenodoRecord.  # noqa: E501
+        :type subject: List[str]
+        :param title: The title of this ZenodoRecord.  # noqa: E501
+        :type title: str
+        :param work: The work of this ZenodoRecord.  # noqa: E501
+        :type work: str
+        """
+        self.openapi_types = {
+            'author': str,
+            'contributor': str,
+            'creator': str,
+            'description': str,
+            'identifier': str,
+            'language': str,
+            'subject': List[str],
+            'title': str,
+            'work': str
+        }
+
+        self.attribute_map = {
+            'author': 'author',
+            'contributor': 'contributor',
+            'creator': 'creator',
+            'description': 'description',
+            'identifier': 'identifier',
+            'language': 'language',
+            'subject': 'subject',
+            'title': 'title',
+            'work': 'work'
+        }
+
+        self._author = author
+        self._contributor = contributor
+        self._creator = creator
+        self._description = description
+        self._identifier = identifier
+        self._language = language
+        self._subject = subject
+        self._title = title
+        self._work = work
+
+    @classmethod
+    def from_dict(cls, dikt) -> 'ZenodoRecord':
+        """Returns the dict as a model
+
+        :param dikt: A dict.
+        :type: dict
+        :return: The ZenodoRecord of this ZenodoRecord.  # noqa: E501
+        :rtype: ZenodoRecord
+        """
+        return util.deserialize_model(dikt, cls)
+
+    @property
+    def author(self):
+        """Gets the author of this ZenodoRecord.
+
+        Author of the text that serves as a basis or target for this record.  # noqa: E501
+
+        :return: The author of this ZenodoRecord.
+        :rtype: str
+        """
+        return self._author
+
+    @author.setter
+    def author(self, author):
+        """Sets the author of this ZenodoRecord.
+
+        Author of the text that serves as a basis or target for this record.  # noqa: E501
+
+        :param author: The author of this ZenodoRecord.
+        :type author: str
+        """
+
+        self._author = author
+
+    @property
+    def contributor(self):
+        """Gets the contributor of this ZenodoRecord.
+
+        People that contributed something to the record.  # noqa: E501
+
+        :return: The contributor of this ZenodoRecord.
+        :rtype: str
+        """
+        return self._contributor
+
+    @contributor.setter
+    def contributor(self, contributor):
+        """Sets the contributor of this ZenodoRecord.
+
+        People that contributed something to the record.  # noqa: E501
+
+        :param contributor: The contributor of this ZenodoRecord.
+        :type contributor: str
+        """
+
+        self._contributor = contributor
+
+    @property
+    def creator(self):
+        """Gets the creator of this ZenodoRecord.
+
+        People that created the record.  # noqa: E501
+
+        :return: The creator of this ZenodoRecord.
+        :rtype: str
+        """
+        return self._creator
+
+    @creator.setter
+    def creator(self, creator):
+        """Sets the creator of this ZenodoRecord.
+
+        People that created the record.  # noqa: E501
+
+        :param creator: The creator of this ZenodoRecord.
+        :type creator: str
+        """
+
+        self._creator = creator
+
+    @property
+    def description(self):
+        """Gets the description of this ZenodoRecord.
+
+        Description of the record.  # noqa: E501
+
+        :return: The description of this ZenodoRecord.
+        :rtype: str
+        """
+        return self._description
+
+    @description.setter
+    def description(self, description):
+        """Sets the description of this ZenodoRecord.
+
+        Description of the record.  # noqa: E501
+
+        :param description: The description of this ZenodoRecord.
+        :type description: str
+        """
+
+        self._description = description
+
+    @property
+    def identifier(self):
+        """Gets the identifier of this ZenodoRecord.
+
+        Identifier for easy access to the record.  # noqa: E501
+
+        :return: The identifier of this ZenodoRecord.
+        :rtype: str
+        """
+        return self._identifier
+
+    @identifier.setter
+    def identifier(self, identifier):
+        """Sets the identifier of this ZenodoRecord.
+
+        Identifier for easy access to the record.  # noqa: E501
+
+        :param identifier: The identifier of this ZenodoRecord.
+        :type identifier: str
+        """
+
+        self._identifier = identifier
+
+    @property
+    def language(self):
+        """Gets the language of this ZenodoRecord.
+
+        Language of the materials in the record.  # noqa: E501
+
+        :return: The language of this ZenodoRecord.
+        :rtype: str
+        """
+        return self._language
+
+    @language.setter
+    def language(self, language):
+        """Sets the language of this ZenodoRecord.
+
+        Language of the materials in the record.  # noqa: E501
+
+        :param language: The language of this ZenodoRecord.
+        :type language: str
+        """
+
+        self._language = language
+
+    @property
+    def subject(self):
+        """Gets the subject of this ZenodoRecord.
+
+
+        :return: The subject of this ZenodoRecord.
+        :rtype: List[str]
+        """
+        return self._subject
+
+    @subject.setter
+    def subject(self, subject):
+        """Sets the subject of this ZenodoRecord.
+
+
+        :param subject: The subject of this ZenodoRecord.
+        :type subject: List[str]
+        """
+
+        self._subject = subject
+
+    @property
+    def title(self):
+        """Gets the title of this ZenodoRecord.
+
+        Title of the record.  # noqa: E501
+
+        :return: The title of this ZenodoRecord.
+        :rtype: str
+        """
+        return self._title
+
+    @title.setter
+    def title(self, title):
+        """Sets the title of this ZenodoRecord.
+
+        Title of the record.  # noqa: E501
+
+        :param title: The title of this ZenodoRecord.
+        :type title: str
+        """
+
+        self._title = title
+
+    @property
+    def work(self):
+        """Gets the work of this ZenodoRecord.
+
+        Name of the work that serves as a basis or target for this record.  # noqa: E501
+
+        :return: The work of this ZenodoRecord.
+        :rtype: str
+        """
+        return self._work
+
+    @work.setter
+    def work(self, work):
+        """Sets the work of this ZenodoRecord.
+
+        Name of the work that serves as a basis or target for this record.  # noqa: E501
+
+        :param work: The work of this ZenodoRecord.
+        :type work: str
+        """
+
+        self._work = work

mc_backend/openapi/openapi_server/openapi/openapi.yaml

@@ -10,14 +10,12 @@ paths:
       operationId: mcserver_app_api_corpus_list_api_get
       parameters:
       - description: Time (in milliseconds) of the last update.
-        explode: true
         in: query
         name: last_update_time
         required: true
         schema:
           example: 123456789
           type: integer
-        style: form
       responses:
         "200":
           content:
@@ -603,6 +601,28 @@ paths:
       summary: Shows how well the vocabulary of a text matches a predefined reference
         vocabulary.
       x-openapi-router-controller: openapi_server.controllers.default_controller
+  /zenodo:
+    get<