Commit e952e195 authored by Konstantin Schulz's avatar Konstantin Schulz

added the Zenodo API for retrieving records with exercise materials from specific communities

parent 63df3a67
Pipeline #16316 passed with stages
in 9 minutes and 23 seconds
......@@ -2,7 +2,7 @@ version: '3.7'
services:
db:
image: postgres
image: postgres:12-alpine
environment:
- POSTGRES_HOST_AUTH_METHOD=trust
# ports:
......
from mcserver import get_app, get_cfg
get_app().run(host=get_cfg().HOST_IP_MCSERVER, port=get_cfg().HOST_PORT, use_reloader=False)
if __name__ == "__main__":
get_app().run(host=get_cfg().HOST_IP_MCSERVER, port=get_cfg().HOST_PORT, use_reloader=False)
......@@ -6,4 +6,4 @@ bp = Blueprint("api", __name__)
api = Api(bp)
from . import corpusAPI, corpusListAPI, exerciseAPI, exerciseListAPI, fileAPI, frequencyAPI, h5pAPI, kwicAPI, \
rawTextAPI, staticExercisesAPI, textcomplexityAPI, validReffAPI, vectorNetworkAPI, vocabularyAPI
rawTextAPI, staticExercisesAPI, textcomplexityAPI, validReffAPI, vectorNetworkAPI, vocabularyAPI, zenodoAPI
"""The Zenodo API. Add it to your REST API to provide users with exercise materials from the Zenodo repository."""
from sickle import Sickle
from typing import List, Set, Any, Dict
import inspect
from sickle.models import Record
from mcserver import Config
from mcserver.app.services import NetworkService
from openapi.openapi_server.models import ZenodoRecord
def get():
"""The GET method for the Zenodo REST API. It provides exercise materials from the Zenodo repository."""
record_members: List[tuple] = inspect.getmembers(ZenodoRecord, lambda a: not (inspect.isroutine(a)))
record_properties: Set[str] = set([x[0] for x in record_members if type(x[1]) == property])
author_property: str = "Autor:"
work_property: str = "Werk:"
sickle: Sickle = Sickle(Config.ZENODO_API_URL)
records: List[Record] = list(sickle.ListRecords(metadataPrefix='oai_dc', set=Config.ZENODO_SET))
zenodo_records: List[ZenodoRecord] = []
for record in records:
md: dict = record.metadata
params: Dict[str, Any] = {rp: md.get(rp, [None])[0] for rp in record_properties}
new_record: ZenodoRecord = ZenodoRecord(**params)
tags: List[str] = md.get("subject", [])
new_record.author = next((x.split(author_property)[-1] for x in tags if x.startswith(author_property)), "")
new_record.work = next((x.split(work_property)[-1] for x in tags if x.startswith(work_property)), "")
zenodo_records.append(new_record)
return NetworkService.make_json_response([x.to_dict() for x in zenodo_records])
......@@ -372,7 +372,7 @@ class CorpusService:
@staticmethod
def update_corpora():
"""Checks the remote repositories for new corpora to be included in our database."""
CorpusService.existing_corpora = db.session.query(Corpus).all()
CorpusService.existing_corpora = DatabaseService.query(Corpus)
resolver: HttpCtsRetriever = HttpCtsRetriever(Config.CTS_API_BASE_URL)
# check the appropriate literature for the desired author
resp: str = resolver.getCapabilities(urn="urn:cts:latinLit") # "urn:cts:greekLit" for Greek
......@@ -400,14 +400,15 @@ class CorpusService:
if urn not in urn_set_existing:
CorpusService.add_corpus(title_value, urn, group_name_value, citation_levels)
else:
corpus_to_update: Corpus = db.session.query(Corpus).filter_by(source_urn=urn).first()
CorpusService.update_corpus(title_value, urn, group_name_value, citation_levels, corpus_to_update)
corpus_to_update: Corpus = DatabaseService.query(Corpus, dict(source_urn=urn), True)
CorpusService.update_corpus(
title_value, urn, group_name_value, citation_levels, corpus_to_update)
for urn in urn_set_existing:
if urn not in urn_set_new:
corpus_to_delete: Corpus = db.session.query(Corpus).filter_by(source_urn=urn).first()
corpus_to_delete: Corpus = DatabaseService.query(Corpus, dict(source_urn=urn), True)
db.session.delete(corpus_to_delete)
db.session.commit()
CorpusService.existing_corpora = db.session.query(Corpus).all()
CorpusService.existing_corpora = DatabaseService.query(Corpus)
db.session.commit()
@staticmethod
......
......@@ -137,6 +137,7 @@ class Config(object):
SERVER_URI_VALID_REFF = SERVER_URI_BASE + "validReff"
SERVER_URI_VECTOR_NETWORK = SERVER_URI_BASE + "vectorNetwork"
SERVER_URI_VOCABULARY = SERVER_URI_BASE + "vocabulary"
SERVER_URI_ZENODO = SERVER_URI_BASE + "zenodo"
# END endpoints
SQLALCHEMY_DATABASE_URI = os.environ.get("DATABASE_URL") or DATABASE_URL_SQLITE
# BEWARE: if True, logs every single database statement executed by this application to STDOUT
......@@ -161,6 +162,8 @@ class Config(object):
VOCABULARY_PROIEL_FILE_NAME = "vocabulary_proiel_treebank.json"
VOCABULARY_VISCHER_FILE_NAME = "vocabulary_vischer.json"
VOCABULARY_VIVA_FILE_NAME = "vocabulary_viva.json"
ZENODO_API_URL = "https://zenodo.org/oai2d"
ZENODO_SET = "user-lexiprojekt-latin-unipotsdam"
class ProductionConfig(Config):
......
......@@ -19,13 +19,7 @@ paths:
schema:
$ref: '../openapi_models.yaml#/components/schemas/Corpus'
parameters:
- name: last_update_time
in: query
description: Time (in milliseconds) of the last update.
required: true
schema:
type: integer
example: 123456789
- $ref: '../openapi_models.yaml#/components/parameters/LastUpdateTimeParam'
/corpora/{cid}:
parameters:
- name: cid
......@@ -232,7 +226,7 @@ paths:
schema:
type: object
description: JSON template for an interactive H5P exercise.
example: {}
example: { }
parameters:
- $ref: '../openapi_models.yaml#/components/parameters/EidParam'
- $ref: '../openapi_models.yaml#/components/parameters/LangParam'
......@@ -386,7 +380,7 @@ paths:
items:
type: array
description: Tokenized sentence.
example: [in, vino, veritas]
example: [ in, vino, veritas ]
items:
type: string
description: Token of a sentence.
......@@ -447,6 +441,19 @@ paths:
application/x-www-form-urlencoded:
schema:
$ref: '../openapi_models.yaml#/components/schemas/VocabularyForm'
/zenodo:
get:
summary: Shows which exercises are available on Zenodo.
operationId: mcserver.app.api.zenodoAPI.get
responses:
"200":
description: Retrieves records from Zenodo communities.
content:
application/json:
schema:
type: array
items:
$ref: '../openapi_models.yaml#/components/schemas/ZenodoRecord'
# include this here so the data model gets generated correctly
components:
schemas:
......
......@@ -11,6 +11,7 @@ from openapi.openapi_server.models.sentence import Sentence # noqa: E501
from openapi.openapi_server.models.static_exercise import StaticExercise # noqa: E501
from openapi.openapi_server.models.text_complexity import TextComplexity # noqa: E501
from openapi.openapi_server.models.vocabulary_mc import VocabularyMC # noqa: E501
from openapi.openapi_server.models.zenodo_record import ZenodoRecord # noqa: E501
from openapi.openapi_server import util
......@@ -349,3 +350,16 @@ def mcserver_app_api_vocabulary_api_post(frequency_upper_bound, query_urn, vocab
if connexion.request.is_json:
vocabulary = VocabularyMC.from_dict(connexion.request.get_json()) # noqa: E501
return 'do some magic!'
def mcserver_app_api_zenodo_api_get(last_update_time): # noqa: E501
"""Shows which exercises are available on Zenodo.
# noqa: E501
:param last_update_time: Time (in milliseconds) of the last update.
:type last_update_time: int
:rtype: List[ZenodoRecord]
"""
return 'do some magic!'
......@@ -28,8 +28,7 @@ from openapi.openapi_server.models.solution import Solution
from openapi.openapi_server.models.solution_element import SolutionElement
from openapi.openapi_server.models.static_exercise import StaticExercise
from openapi.openapi_server.models.text_complexity import TextComplexity
from openapi.openapi_server.models.text_complexity_form import TextComplexityForm
from openapi.openapi_server.models.text_complexity_form_extension import TextComplexityFormExtension
from openapi.openapi_server.models.vector_network_form import VectorNetworkForm
from openapi.openapi_server.models.vocabulary_form import VocabularyForm
from openapi.openapi_server.models.vocabulary_mc import VocabularyMC
from openapi.openapi_server.models.zenodo_record import ZenodoRecord
# coding: utf-8
from __future__ import absolute_import
from datetime import date, datetime # noqa: F401
from typing import List, Dict # noqa: F401
from openapi.openapi_server.models.base_model_ import Model
from openapi.openapi_server import util
class Record(Model):
"""NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
Do not edit the class manually.
"""
def __init__(self, description=None, title=None): # noqa: E501
"""Record - a model defined in OpenAPI
:param description: The description of this Record. # noqa: E501
:type description: str
:param title: The title of this Record. # noqa: E501
:type title: str
"""
self.openapi_types = {
'description': str,
'title': str
}
self.attribute_map = {
'description': 'description',
'title': 'title'
}
self._description = description
self._title = title
@classmethod
def from_dict(cls, dikt) -> 'Record':
"""Returns the dict as a model
:param dikt: A dict.
:type: dict
:return: The Record of this Record. # noqa: E501
:rtype: Record
"""
return util.deserialize_model(dikt, cls)
@property
def description(self):
"""Gets the description of this Record.
Description of the record. # noqa: E501
:return: The description of this Record.
:rtype: str
"""
return self._description
@description.setter
def description(self, description):
"""Sets the description of this Record.
Description of the record. # noqa: E501
:param description: The description of this Record.
:type description: str
"""
self._description = description
@property
def title(self):
"""Gets the title of this Record.
Title of the record. # noqa: E501
:return: The title of this Record.
:rtype: str
"""
return self._title
@title.setter
def title(self, title):
"""Sets the title of this Record.
Title of the record. # noqa: E501
:param title: The title of this Record.
:type title: str
"""
self._title = title
# coding: utf-8
from __future__ import absolute_import
from datetime import date, datetime # noqa: F401
from typing import List, Dict # noqa: F401
from openapi.openapi_server.models.base_model_ import Model
from openapi.openapi_server import util
class ZenodoRecord(Model):
"""NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
Do not edit the class manually.
"""
def __init__(self, author=None, contributor=None, creator=None, description=None, identifier=None, language=None, subject=None, title=None, work=None): # noqa: E501
"""ZenodoRecord - a model defined in OpenAPI
:param author: The author of this ZenodoRecord. # noqa: E501
:type author: str
:param contributor: The contributor of this ZenodoRecord. # noqa: E501
:type contributor: str
:param creator: The creator of this ZenodoRecord. # noqa: E501
:type creator: str
:param description: The description of this ZenodoRecord. # noqa: E501
:type description: str
:param identifier: The identifier of this ZenodoRecord. # noqa: E501
:type identifier: str
:param language: The language of this ZenodoRecord. # noqa: E501
:type language: str
:param subject: The subject of this ZenodoRecord. # noqa: E501
:type subject: List[str]
:param title: The title of this ZenodoRecord. # noqa: E501
:type title: str
:param work: The work of this ZenodoRecord. # noqa: E501
:type work: str
"""
self.openapi_types = {
'author': str,
'contributor': str,
'creator': str,
'description': str,
'identifier': str,
'language': str,
'subject': List[str],
'title': str,
'work': str
}
self.attribute_map = {
'author': 'author',
'contributor': 'contributor',
'creator': 'creator',
'description': 'description',
'identifier': 'identifier',
'language': 'language',
'subject': 'subject',
'title': 'title',
'work': 'work'
}
self._author = author
self._contributor = contributor
self._creator = creator
self._description = description
self._identifier = identifier
self._language = language
self._subject = subject
self._title = title
self._work = work
@classmethod
def from_dict(cls, dikt) -> 'ZenodoRecord':
"""Returns the dict as a model
:param dikt: A dict.
:type: dict
:return: The ZenodoRecord of this ZenodoRecord. # noqa: E501
:rtype: ZenodoRecord
"""
return util.deserialize_model(dikt, cls)
@property
def author(self):
"""Gets the author of this ZenodoRecord.
Author of the text that serves as a basis or target for this record. # noqa: E501
:return: The author of this ZenodoRecord.
:rtype: str
"""
return self._author
@author.setter
def author(self, author):
"""Sets the author of this ZenodoRecord.
Author of the text that serves as a basis or target for this record. # noqa: E501
:param author: The author of this ZenodoRecord.
:type author: str
"""
self._author = author
@property
def contributor(self):
"""Gets the contributor of this ZenodoRecord.
People that contributed something to the record. # noqa: E501
:return: The contributor of this ZenodoRecord.
:rtype: str
"""
return self._contributor
@contributor.setter
def contributor(self, contributor):
"""Sets the contributor of this ZenodoRecord.
People that contributed something to the record. # noqa: E501
:param contributor: The contributor of this ZenodoRecord.
:type contributor: str
"""
self._contributor = contributor
@property
def creator(self):
"""Gets the creator of this ZenodoRecord.
People that created the record. # noqa: E501
:return: The creator of this ZenodoRecord.
:rtype: str
"""
return self._creator
@creator.setter
def creator(self, creator):
"""Sets the creator of this ZenodoRecord.
People that created the record. # noqa: E501
:param creator: The creator of this ZenodoRecord.
:type creator: str
"""
self._creator = creator
@property
def description(self):
"""Gets the description of this ZenodoRecord.
Description of the record. # noqa: E501
:return: The description of this ZenodoRecord.
:rtype: str
"""
return self._description
@description.setter
def description(self, description):
"""Sets the description of this ZenodoRecord.
Description of the record. # noqa: E501
:param description: The description of this ZenodoRecord.
:type description: str
"""
self._description = description
@property
def identifier(self):
"""Gets the identifier of this ZenodoRecord.
Identifier for easy access to the record. # noqa: E501
:return: The identifier of this ZenodoRecord.
:rtype: str
"""
return self._identifier
@identifier.setter
def identifier(self, identifier):
"""Sets the identifier of this ZenodoRecord.
Identifier for easy access to the record. # noqa: E501
:param identifier: The identifier of this ZenodoRecord.
:type identifier: str
"""
self._identifier = identifier
@property
def language(self):
"""Gets the language of this ZenodoRecord.
Language of the materials in the record. # noqa: E501
:return: The language of this ZenodoRecord.
:rtype: str
"""
return self._language
@language.setter
def language(self, language):
"""Sets the language of this ZenodoRecord.
Language of the materials in the record. # noqa: E501
:param language: The language of this ZenodoRecord.
:type language: str
"""
self._language = language
@property
def subject(self):
"""Gets the subject of this ZenodoRecord.
:return: The subject of this ZenodoRecord.
:rtype: List[str]
"""
return self._subject
@subject.setter
def subject(self, subject):
"""Sets the subject of this ZenodoRecord.
:param subject: The subject of this ZenodoRecord.
:type subject: List[str]
"""
self._subject = subject
@property
def title(self):
"""Gets the title of this ZenodoRecord.
Title of the record. # noqa: E501
:return: The title of this ZenodoRecord.
:rtype: str
"""
return self._title
@title.setter
def title(self, title):
"""Sets the title of this ZenodoRecord.
Title of the record. # noqa: E501
:param title: The title of this ZenodoRecord.
:type title: str
"""
self._title = title
@property
def work(self):
"""Gets the work of this ZenodoRecord.
Name of the work that serves as a basis or target for this record. # noqa: E501
:return: The work of this ZenodoRecord.
:rtype: str
"""
return self._work
@work.setter
def work(self, work):
"""Sets the work of this ZenodoRecord.
Name of the work that serves as a basis or target for this record. # noqa: E501
:param work: The work of this ZenodoRecord.
:type work: str
"""
self._work = work
......@@ -10,14 +10,12 @@ paths:
operationId: mcserver_app_api_corpus_list_api_get
parameters:
- description: Time (in milliseconds) of the last update.
explode: true
in: query
name: last_update_time
required: true
schema:
example: 123456789
type: integer
style: form
responses:
"200":
content:
......@@ -603,6 +601,28 @@ paths:
summary: Shows how well the vocabulary of a text matches a predefined reference
vocabulary.
x-openapi-router-controller: openapi_server.controllers.default_controller
/zenodo:
get<