Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
What's new
7
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in / Register
Toggle navigation
Open sidebar
callidus
Machina Callida
Commits
57b321d2
Commit
57b321d2
authored
Jun 30, 2020
by
Konstantin Schulz
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
commits to the database are now handled more uniformly
parent
aa35072d
Pipeline
#12195
failed with stages
in 2 minutes and 38 seconds
Changes
10
Pipelines
2
Hide whitespace changes
Inline
Side-by-side
Showing
10 changed files
with
95 additions
and
85 deletions
+95
-85
mc_backend/mcserver/app/__init__.py
mc_backend/mcserver/app/__init__.py
+0
-1
mc_backend/mcserver/app/api/corpusAPI.py
mc_backend/mcserver/app/api/corpusAPI.py
+3
-3
mc_backend/mcserver/app/api/corpusListAPI.py
mc_backend/mcserver/app/api/corpusListAPI.py
+4
-9
mc_backend/mcserver/app/api/exerciseAPI.py
mc_backend/mcserver/app/api/exerciseAPI.py
+5
-4
mc_backend/mcserver/app/api/exerciseListAPI.py
mc_backend/mcserver/app/api/exerciseListAPI.py
+3
-3
mc_backend/mcserver/app/api/fileAPI.py
mc_backend/mcserver/app/api/fileAPI.py
+5
-5
mc_backend/mcserver/app/api/h5pAPI.py
mc_backend/mcserver/app/api/h5pAPI.py
+2
-2
mc_backend/mcserver/app/services/databaseService.py
mc_backend/mcserver/app/services/databaseService.py
+18
-10
mc_backend/mocks.py
mc_backend/mocks.py
+17
-23
mc_backend/tests.py
mc_backend/tests.py
+38
-25
No files found.
mc_backend/mcserver/app/__init__.py
View file @
57b321d2
...
...
@@ -3,7 +3,6 @@ import logging
import
os
import
sys
from
logging.handlers
import
RotatingFileHandler
from
pathlib
import
Path
from
threading
import
Thread
from
time
import
strftime
from
typing
import
Type
...
...
mc_backend/mcserver/app/api/corpusAPI.py
View file @
57b321d2
...
...
@@ -7,7 +7,7 @@ from flask import Response
from
mcserver
import
Config
from
mcserver.app
import
db
from
mcserver.app.services
import
NetworkService
from
mcserver.app.services
import
NetworkService
,
DatabaseService
from
mcserver.models_auto
import
Corpus
...
...
@@ -17,7 +17,7 @@ def delete(cid: int) -> Union[Response, ConnexionResponse]:
if
corpus
is
None
:
return
connexion
.
problem
(
404
,
Config
.
ERROR_TITLE_NOT_FOUND
,
Config
.
ERROR_MESSAGE_CORPUS_NOT_FOUND
)
db
.
session
.
delete
(
corpus
)
db
.
session
.
commit
()
DatabaseService
.
commit
()
return
NetworkService
.
make_json_response
(
True
)
...
...
@@ -37,5 +37,5 @@ def patch(cid: int, **kwargs) -> Union[Response, ConnexionResponse]:
for
k
,
v
in
kwargs
.
items
():
if
v
is
not
None
:
setattr
(
corpus
,
k
,
v
)
db
.
session
.
commit
()
DatabaseService
.
commit
()
return
NetworkService
.
make_json_response
(
corpus
.
to_dict
())
mc_backend/mcserver/app/api/corpusListAPI.py
View file @
57b321d2
"""The corpus list API. Add it to your REST API to provide users with a list of metadata for available texts."""
from
connexion.lifecycle
import
ConnexionResponse
from
flask
import
Response
from
sqlalchemy.exc
import
OperationalError
,
InvalidRequestError
from
typing
import
List
,
Union
from
mcserver.app
import
db
from
mcserver.app.models
import
ResourceType
from
mcserver.app.services
import
NetworkService
from
mcserver.app.services
import
NetworkService
,
DatabaseService
from
mcserver.models_auto
import
Corpus
,
UpdateInfo
def
get
(
last_update_time
:
int
)
->
Union
[
Response
,
ConnexionResponse
]:
"""The GET method for the corpus list REST API. It provides metadata for all available texts."""
ui_cts
:
UpdateInfo
try
:
ui_cts
=
db
.
session
.
query
(
UpdateInfo
).
filter_by
(
resource_type
=
ResourceType
.
cts_data
.
name
).
first
()
db
.
session
.
commit
()
except
(
InvalidRequestError
,
OperationalError
):
db
.
session
.
rollback
()
return
NetworkService
.
make_json_response
(
None
)
ui_cts
=
db
.
session
.
query
(
UpdateInfo
).
filter_by
(
resource_type
=
ResourceType
.
cts_data
.
name
).
first
()
DatabaseService
.
commit
()
if
ui_cts
.
last_modified_time
>=
last_update_time
/
1000
:
corpora
:
List
[
Corpus
]
=
db
.
session
.
query
(
Corpus
).
all
()
db
.
session
.
commit
()
DatabaseService
.
commit
()
return
NetworkService
.
make_json_response
([
x
.
to_dict
()
for
x
in
corpora
])
return
NetworkService
.
make_json_response
(
None
)
mc_backend/mcserver/app/api/exerciseAPI.py
View file @
57b321d2
...
...
@@ -9,7 +9,8 @@ from flask import Response
from
mcserver.app
import
db
from
mcserver.app.models
import
ExerciseType
,
Solution
,
ExerciseData
,
AnnisResponse
,
Phenomenon
,
TextComplexity
,
\
TextComplexityMeasure
,
ResourceType
,
ExerciseMC
,
GraphData
from
mcserver.app.services
import
AnnotationService
,
CorpusService
,
NetworkService
,
TextComplexityService
from
mcserver.app.services
import
AnnotationService
,
CorpusService
,
NetworkService
,
TextComplexityService
,
\
DatabaseService
from
mcserver.config
import
Config
from
mcserver.models_auto
import
Exercise
,
TExercise
,
UpdateInfo
from
openapi.openapi_server.models
import
ExerciseForm
...
...
@@ -28,14 +29,14 @@ def adjust_solutions(exercise_data: ExerciseData, exercise_type: str, solutions:
def
get
(
eid
:
str
)
->
Union
[
Response
,
ConnexionResponse
]:
exercise
:
TExercise
=
db
.
session
.
query
(
Exercise
).
filter_by
(
eid
=
eid
).
first
()
db
.
session
.
commit
()
DatabaseService
.
commit
()
if
exercise
is
None
:
return
connexion
.
problem
(
404
,
Config
.
ERROR_TITLE_NOT_FOUND
,
Config
.
ERROR_MESSAGE_EXERCISE_NOT_FOUND
)
ar
:
AnnisResponse
=
CorpusService
.
get_corpus
(
cts_urn
=
exercise
.
urn
,
is_csm
=
False
)
if
not
ar
.
graph_data
.
nodes
:
return
connexion
.
problem
(
404
,
Config
.
ERROR_TITLE_NOT_FOUND
,
Config
.
ERROR_MESSAGE_CORPUS_NOT_FOUND
)
exercise
.
last_access_time
=
datetime
.
utcnow
().
timestamp
()
db
.
session
.
commit
()
DatabaseService
.
commit
()
exercise_type
:
ExerciseType
=
ExerciseType
(
exercise
.
exercise_type
)
ar
.
solutions
=
json
.
loads
(
exercise
.
solutions
)
ar
.
uri
=
NetworkService
.
get_exercise_uri
(
exercise
)
...
...
@@ -109,7 +110,7 @@ def map_exercise_data_to_database(exercise_data: ExerciseData, exercise_type: st
ui_exercises
:
UpdateInfo
=
db
.
session
.
query
(
UpdateInfo
).
filter_by
(
resource_type
=
ResourceType
.
exercise_list
.
name
).
first
()
ui_exercises
.
last_modified_time
=
datetime
.
utcnow
().
timestamp
()
db
.
session
.
commit
()
DatabaseService
.
commit
()
return
new_exercise
...
...
mc_backend/mcserver/app/api/exerciseListAPI.py
View file @
57b321d2
...
...
@@ -5,7 +5,7 @@ import conllu
from
conllu
import
TokenList
from
mcserver.app
import
db
from
mcserver.app.models
import
Language
,
VocabularyCorpus
,
ResourceType
from
mcserver.app.services
import
NetworkService
,
FileService
from
mcserver.app.services
import
NetworkService
,
FileService
,
DatabaseService
from
mcserver.models_auto
import
Exercise
,
UpdateInfo
from
openapi.openapi_server.models
import
MatchingExercise
...
...
@@ -15,7 +15,7 @@ def get(lang: str, frequency_upper_bound: int, last_update_time: int, vocabulary
vocabulary_set
:
Set
[
str
]
ui_exercises
:
UpdateInfo
=
db
.
session
.
query
(
UpdateInfo
).
filter_by
(
resource_type
=
ResourceType
.
exercise_list
.
name
).
first
()
db
.
session
.
commit
()
DatabaseService
.
commit
()
if
ui_exercises
.
last_modified_time
<
last_update_time
/
1000
:
return
NetworkService
.
make_json_response
([])
try
:
...
...
@@ -29,7 +29,7 @@ def get(lang: str, frequency_upper_bound: int, last_update_time: int, vocabulary
except
ValueError
:
lang
=
Language
.
English
exercises
:
List
[
Exercise
]
=
db
.
session
.
query
(
Exercise
).
filter_by
(
language
=
lang
.
value
)
db
.
session
.
commit
()
DatabaseService
.
commit
()
matching_exercises
:
List
[
MatchingExercise
]
=
[
MatchingExercise
.
from_dict
(
x
.
to_dict
())
for
x
in
exercises
]
if
len
(
vocabulary_set
):
for
exercise
in
matching_exercises
:
...
...
mc_backend/mcserver/app/api/fileAPI.py
View file @
57b321d2
...
...
@@ -10,7 +10,7 @@ from flask import send_from_directory, Response
from
werkzeug.wrappers
import
ETagResponseMixin
from
mcserver.app
import
db
from
mcserver.app.models
import
FileType
,
ResourceType
,
DownloadableFile
,
MimeType
,
XapiStatement
,
LearningResultMC
from
mcserver.app.services
import
FileService
,
NetworkService
from
mcserver.app.services
import
FileService
,
NetworkService
,
DatabaseService
from
mcserver.config
import
Config
from
mcserver.models_auto
import
Exercise
,
UpdateInfo
,
LearningResult
...
...
@@ -29,14 +29,14 @@ def clean_tmp_folder():
FileService
.
downloadable_files
.
remove
(
file_to_delete
)
os
.
remove
(
os
.
path
.
join
(
Config
.
TMP_DIRECTORY
,
file
))
ui_file
.
last_modified_time
=
datetime
.
utcnow
().
timestamp
()
db
.
session
.
commit
()
DatabaseService
.
commit
()
def
get
(
id
:
str
,
type
:
FileType
,
solution_indices
:
List
[
int
])
->
Union
[
ETagResponseMixin
,
ConnexionResponse
]:
"""The GET method for the file REST API. It provides the URL to download a specific file."""
clean_tmp_folder
()
exercise
:
Exercise
=
db
.
session
.
query
(
Exercise
).
filter_by
(
eid
=
id
).
first
()
db
.
session
.
commit
()
DatabaseService
.
commit
()
file_name
:
str
=
id
+
"."
+
str
(
type
)
mime_type
:
str
=
MimeType
[
type
].
value
if
exercise
is
None
:
...
...
@@ -45,7 +45,7 @@ def get(id: str, type: FileType, solution_indices: List[int]) -> Union[ETagRespo
return
connexion
.
problem
(
404
,
Config
.
ERROR_TITLE_NOT_FOUND
,
Config
.
ERROR_MESSAGE_EXERCISE_NOT_FOUND
)
return
send_from_directory
(
Config
.
TMP_DIRECTORY
,
file_name
,
mimetype
=
mime_type
,
as_attachment
=
True
)
exercise
.
last_access_time
=
datetime
.
utcnow
().
timestamp
()
db
.
session
.
commit
()
DatabaseService
.
commit
()
if
solution_indices
:
file_name
=
id
+
"-"
+
str
(
uuid
.
uuid4
())
+
"."
+
str
(
type
)
existing_file
:
DownloadableFile
=
next
(
...
...
@@ -101,5 +101,5 @@ def save_learning_result(xapi_statement: XapiStatement) -> LearningResult:
verb_display
=
xapi_statement
.
verb
.
display
.
en_us
)
db
.
session
.
add
(
learning_result
)
db
.
session
.
commit
()
DatabaseService
.
commit
()
return
learning_result
mc_backend/mcserver/app/api/h5pAPI.py
View file @
57b321d2
...
...
@@ -5,7 +5,7 @@ from flask import Response
from
mcserver
import
Config
from
mcserver.app
import
db
from
mcserver.app.models
import
Language
,
ExerciseType
,
Solution
from
mcserver.app.services
import
TextService
,
NetworkService
from
mcserver.app.services
import
TextService
,
NetworkService
,
DatabaseService
from
mcserver.models_auto
import
Exercise
...
...
@@ -17,7 +17,7 @@ def get(eid: str, lang: str, solution_indices: List[int]) -> Union[Response, Con
except
ValueError
:
language
=
Language
.
English
exercise
:
Exercise
=
db
.
session
.
query
(
Exercise
).
filter_by
(
eid
=
eid
).
first
()
db
.
session
.
commit
()
DatabaseService
.
commit
()
if
exercise
is
None
:
return
connexion
.
problem
(
404
,
Config
.
ERROR_TITLE_NOT_FOUND
,
Config
.
ERROR_MESSAGE_EXERCISE_NOT_FOUND
)
text_field_content
:
str
=
""
...
...
mc_backend/mcserver/app/services/databaseService.py
View file @
57b321d2
...
...
@@ -4,11 +4,10 @@ from typing import List, Dict
from
flask
import
Flask
from
flask_migrate
import
stamp
,
upgrade
import
rapidjson
as
json
from
sqlalchemy.exc
import
OperationalError
from
sqlalchemy.exc
import
OperationalError
,
InvalidRequestError
from
mcserver.app
import
db
from
mcserver.app.models
import
CitationLevel
,
ResourceType
,
TextComplexityMeasure
,
AnnisResponse
,
GraphData
,
\
TextComplexity
from
mcserver.app.models
import
CitationLevel
,
ResourceType
,
TextComplexityMeasure
,
AnnisResponse
,
TextComplexity
from
mcserver.app.services
import
CorpusService
,
CustomCorpusService
,
TextComplexityService
from
mcserver.config
import
Config
from
mcserver.models_auto
import
Corpus
,
Exercise
,
UpdateInfo
...
...
@@ -21,7 +20,7 @@ class DatabaseService:
""" Checks whether the corpus list needs to be updated. If yes, it performs the update. """
app
.
logger
.
info
(
"Corpus update started."
)
ui_cts
:
UpdateInfo
=
db
.
session
.
query
(
UpdateInfo
).
filter_by
(
resource_type
=
ResourceType
.
cts_data
.
name
).
first
()
db
.
session
.
commit
()
DatabaseService
.
commit
()
if
ui_cts
is
None
:
app
.
logger
.
info
(
"UpdateInfo not available!"
)
return
...
...
@@ -30,9 +29,18 @@ class DatabaseService:
if
(
datetime
.
utcnow
()
-
ui_datetime
).
total_seconds
()
>
Config
.
INTERVAL_CORPUS_UPDATE
:
CorpusService
.
update_corpora
()
ui_cts
.
last_modified_time
=
datetime
.
utcnow
().
timestamp
()
db
.
session
.
commit
()
DatabaseService
.
commit
()
app
.
logger
.
info
(
"Corpus update completed."
)
@
staticmethod
def
commit
():
"""Commits the last action to the database and, if it fails, rolls back the current session."""
try
:
db
.
session
.
commit
()
except
(
OperationalError
,
InvalidRequestError
):
db
.
session
.
rollback
()
raise
@
staticmethod
def
init_db_alembic
()
->
None
:
"""In Docker, the alembic version is not initially written to the database, so we need to set it manually."""
...
...
@@ -45,7 +53,7 @@ class DatabaseService:
"""Initializes the corpus list if it is not already there and up to date."""
if
db
.
engine
.
dialect
.
has_table
(
db
.
engine
,
Config
.
DATABASE_TABLE_CORPUS
):
CorpusService
.
existing_corpora
=
db
.
session
.
query
(
Corpus
).
all
()
db
.
session
.
commit
()
DatabaseService
.
commit
()
urn_dict
:
Dict
[
str
,
int
]
=
{
v
.
source_urn
:
i
for
i
,
v
in
enumerate
(
CorpusService
.
existing_corpora
)}
for
cc
in
CustomCorpusService
.
custom_corpora
:
if
cc
.
corpus
.
source_urn
in
urn_dict
:
...
...
@@ -62,7 +70,7 @@ class DatabaseService:
group_name_value
=
cc
.
corpus
.
author
,
citation_levels
=
citation_levels
)
CorpusService
.
existing_corpora
=
db
.
session
.
query
(
Corpus
).
all
()
db
.
session
.
commit
()
DatabaseService
.
commit
()
@
staticmethod
def
init_db_update_info
()
->
None
:
...
...
@@ -74,7 +82,7 @@ class DatabaseService:
ui_cts
=
UpdateInfo
.
from_dict
(
resource_type
=
rt
.
name
,
last_modified_time
=
1
,
created_time
=
datetime
.
utcnow
().
timestamp
())
db
.
session
.
add
(
ui_cts
)
db
.
session
.
commit
()
DatabaseService
.
commit
()
@
staticmethod
def
init_updater
(
app
:
Flask
)
->
None
:
...
...
@@ -104,11 +112,11 @@ class DatabaseService:
if
(
now
-
exercise_datetime
).
total_seconds
()
>
Config
.
INTERVAL_EXERCISE_DELETE
or
\
not
exercise
.
urn
or
not
json
.
loads
(
exercise
.
solutions
):
db
.
session
.
delete
(
exercise
)
db
.
session
.
commit
()
DatabaseService
.
commit
()
# manually add text complexity measures for old exercises
elif
not
exercise
.
text_complexity
:
ar
:
AnnisResponse
=
CorpusService
.
get_corpus
(
exercise
.
urn
,
is_csm
=
is_csm
)
tc
:
TextComplexity
=
TextComplexityService
.
text_complexity
(
TextComplexityMeasure
.
all
.
name
,
exercise
.
urn
,
is_csm
,
ar
.
graph_data
)
exercise
.
text_complexity
=
tc
.
all
db
.
session
.
commit
()
DatabaseService
.
commit
()
mc_backend/mocks.py
View file @
57b321d2
...
...
@@ -15,39 +15,44 @@ from gensim.models import Word2Vec
from
gensim.models.keyedvectors
import
Vocab
from
networkx
import
Graph
from
numpy.core.multiarray
import
ndarray
from
sqlalchemy.exc
import
OperationalError
from
mcserver
import
Config
,
TestingConfig
from
mcserver.app
import
db
,
shutdown_session
from
mcserver.app.models
import
Phenomenon
,
PartOfSpeech
,
CitationLevel
,
ExerciseData
,
GraphData
,
\
LinkMC
,
NodeMC
,
Language
,
Dependency
,
Case
,
AnnisResponse
,
Solution
,
TextPart
,
Citation
,
ExerciseMC
,
CorpusMC
,
\
SolutionElement
from
mcserver.app.services
import
AnnotationService
,
CustomCorpusService
,
TextService
from
mcserver.app.services
import
AnnotationService
,
CustomCorpusService
,
TextService
,
DatabaseService
from
mcserver.models_auto
import
Corpus
,
Exercise
,
UpdateInfo
class
MockFilterBy
:
def
__init__
(
self
,
do_raise
:
bool
=
False
,
ui
:
UpdateInfo
=
None
):
self
.
do_raise
:
bool
=
do_raise
def
__init__
(
self
,
ui
:
UpdateInfo
=
None
):
self
.
ui
:
UpdateInfo
=
ui
def
first
(
self
):
if
self
.
do_raise
:
raise
OperationalError
(
"error"
,
[],
""
)
else
:
return
self
.
ui
return
self
.
ui
class
MockQuery
:
def
__init__
(
self
,
do_raise
:
bool
=
False
,
ui
:
UpdateInfo
=
None
):
self
.
do_raise
:
bool
=
do_raise
def
__init__
(
self
,
ui
:
UpdateInfo
=
None
):
self
.
ui
:
UpdateInfo
=
ui
def
all
(
self
):
return
db
.
session
.
query
(
Corpus
).
all
()
def
filter_by
(
self
,
**
kwargs
):
return
MockFilterBy
(
self
.
do_raise
,
self
.
ui
)
return
MockFilterBy
(
self
.
ui
)
class
MockResponse
:
def
__init__
(
self
,
text
:
str
,
ok
:
bool
=
True
,
content
:
bytes
=
b
""
):
self
.
content
:
bytes
=
content
self
.
encoding
:
str
=
"utf-8"
self
.
ok
:
bool
=
ok
self
.
text
:
str
=
text
def
raise_for_status
(
self
)
->
None
:
pass
class
MockWV
:
...
...
@@ -66,17 +71,6 @@ class MockW2V:
self
.
wv
=
MockWV
()
class
MockResponse
:
def
__init__
(
self
,
text
:
str
,
ok
:
bool
=
True
,
content
:
bytes
=
b
""
):
self
.
content
:
bytes
=
content
self
.
encoding
:
str
=
"utf-8"
self
.
ok
:
bool
=
ok
self
.
text
:
str
=
text
def
raise_for_status
(
self
)
->
None
:
pass
class
TestHelper
:
def
__init__
(
self
,
app
:
Flask
):
self
.
app
:
Flask
=
app
...
...
@@ -100,7 +94,7 @@ class TestHelper:
Mocks
.
app_dict
[
class_name
]
=
TestHelper
(
app_factory
(
TestingConfig
))
Mocks
.
app_dict
[
class_name
].
app
.
logger
.
setLevel
(
logging
.
WARNING
)
Mocks
.
app_dict
[
class_name
].
app
.
testing
=
True
db
.
session
.
commit
()
DatabaseService
.
commit
()
class
Mocks
:
...
...
mc_backend/tests.py
View file @
57b321d2
...
...
@@ -88,7 +88,7 @@ class McTestCase(unittest.TestCase):
ui_cts
:
UpdateInfo
=
UpdateInfo
.
from_dict
(
resource_type
=
ResourceType
.
cts_data
.
name
,
last_modified_time
=
datetime
.
utcnow
().
timestamp
(),
created_time
=
1
)
db
.
session
.
add
(
ui_cts
)
db
.
session
.
commit
()
DatabaseService
.
commit
()
@
staticmethod
def
clear_folder
(
folder_path
:
str
):
...
...
@@ -135,11 +135,11 @@ class McTestCase(unittest.TestCase):
def
test_api_corpus_list_get
(
self
):
"""Adds multiple texts to the database and queries them all."""
def
expect_result
(
self
:
McTestCase
,
mock
:
MagicMock
,
do_raise
:
bool
,
lut
:
str
,
result
:
Any
,
def
expect_result
(
self
:
McTestCase
,
mock
:
MagicMock
,
lut
:
str
,
result
:
Any
,
lmt
:
datetime
=
datetime
.
utcnow
()):
ui
:
UpdateInfo
=
UpdateInfo
.
from_dict
(
resource_type
=
ResourceType
.
cts_data
.
name
,
last_modified_time
=
lmt
.
timestamp
(),
created_time
=
1
)
mock
.
session
.
query
.
return_value
=
MockQuery
(
do_raise
,
ui
)
mock
.
session
.
query
.
return_value
=
MockQuery
(
ui
)
response
:
Response
=
Mocks
.
app_dict
[
self
.
class_name
].
client
.
get
(
TestingConfig
.
SERVER_URI_CORPORA
,
query_string
=
dict
(
last_update_time
=
lut
))
data_json
=
json
.
loads
(
response
.
get_data
())
...
...
@@ -148,12 +148,11 @@ class McTestCase(unittest.TestCase):
self
.
assertEqual
(
data_json
,
result
)
with
patch
.
object
(
mcserver
.
app
.
api
.
corpusListAPI
,
"db"
)
as
mock_db
:
expect_result
(
self
,
mock_db
,
True
,
"0"
,
None
)
expect_result
(
self
,
mock_db
,
False
,
str
(
int
(
datetime
.
utcnow
().
timestamp
()
*
1000
)),
None
,
expect_result
(
self
,
mock_db
,
str
(
int
(
datetime
.
utcnow
().
timestamp
()
*
1000
)),
None
,
datetime
.
fromtimestamp
(
0
))
db
.
session
.
add_all
(
Mocks
.
corpora
)
db
.
session
.
commit
()
expect_result
(
self
,
mock_db
,
False
,
"0"
,
Mocks
.
corpora
,
datetime
.
fromtimestamp
(
time
.
time
()))
DatabaseService
.
commit
()
expect_result
(
self
,
mock_db
,
"0"
,
Mocks
.
corpora
,
datetime
.
fromtimestamp
(
time
.
time
()))
db
.
session
.
query
(
Corpus
).
delete
()
db
.
session
.
query
(
UpdateInfo
).
delete
()
# dirty hack so we can reuse it in other tests
...
...
@@ -187,14 +186,14 @@ class McTestCase(unittest.TestCase):
old_urn
:
str
=
Mocks
.
exercise
.
urn
Mocks
.
exercise
.
urn
=
""
db
.
session
.
add
(
Mocks
.
exercise
)
db
.
session
.
commit
()
DatabaseService
.
commit
()
ar
:
AnnisResponse
=
AnnisResponse
(
solutions
=
[],
graph_data
=
GraphData
(
links
=
[],
nodes
=
[]))
with
patch
.
object
(
CorpusService
,
"get_corpus"
,
side_effect
=
[
ar
,
Mocks
.
annis_response
]):
response
=
Mocks
.
app_dict
[
self
.
class_name
].
client
.
get
(
Config
.
SERVER_URI_EXERCISE
,
query_string
=
dict
(
eid
=
Mocks
.
exercise
.
eid
))
self
.
assertEqual
(
response
.
status_code
,
404
)
Mocks
.
exercise
.
urn
=
old_urn
db
.
session
.
commit
()
DatabaseService
.
commit
()
response
=
Mocks
.
app_dict
[
self
.
class_name
].
client
.
get
(
Config
.
SERVER_URI_EXERCISE
,
query_string
=
dict
(
eid
=
Mocks
.
exercise
.
eid
))
graph_dict
:
dict
=
json
.
loads
(
response
.
get_data
(
as_text
=
True
))
...
...
@@ -219,7 +218,7 @@ class McTestCase(unittest.TestCase):
ui_exercises
:
UpdateInfo
=
UpdateInfo
.
from_dict
(
resource_type
=
ResourceType
.
exercise_list
.
name
,
last_modified_time
=
1
,
created_time
=
1
)
db
.
session
.
add
(
ui_exercises
)
db
.
session
.
commit
()
DatabaseService
.
commit
()
ef
:
ExerciseForm
=
ExerciseForm
(
urn
=
Mocks
.
exercise
.
urn
,
type
=
ExerciseType
.
matching
.
value
,
search_values
=
Mocks
.
exercise
.
search_values
,
instructions
=
'abc'
)
with
patch
.
object
(
mcserver
.
app
.
api
.
exerciseAPI
.
requests
,
"post"
,
side_effect
=
post_response
):
...
...
@@ -240,14 +239,14 @@ class McTestCase(unittest.TestCase):
ui_exercises
:
UpdateInfo
=
UpdateInfo
.
from_dict
(
resource_type
=
ResourceType
.
exercise_list
.
name
,
last_modified_time
=
1
,
created_time
=
1
)
db
.
session
.
add
(
ui_exercises
)
db
.
session
.
commit
()
DatabaseService
.
commit
()
args
:
dict
=
dict
(
lang
=
"fr"
,
last_update_time
=
int
(
time
.
time
()))
response
:
Response
=
Mocks
.
app_dict
[
self
.
class_name
].
client
.
get
(
TestingConfig
.
SERVER_URI_EXERCISE_LIST
,
query_string
=
args
)
self
.
assertEqual
(
json
.
loads
(
response
.
get_data
()),
[])
args
[
"last_update_time"
]
=
0
db
.
session
.
add
(
Mocks
.
exercise
)
db
.
session
.
commit
()
DatabaseService
.
commit
()
response
=
Mocks
.
app_dict
[
self
.
class_name
].
client
.
get
(
TestingConfig
.
SERVER_URI_EXERCISE_LIST
,
query_string
=
args
)
exercises
:
List
[
MatchingExercise
]
=
[]
for
exercise_dict
in
json
.
loads
(
response
.
get_data
(
as_text
=
True
)):
...
...
@@ -268,7 +267,7 @@ class McTestCase(unittest.TestCase):
ui_file
:
UpdateInfo
=
UpdateInfo
.
from_dict
(
resource_type
=
ResourceType
.
file_api_clean
.
name
,
last_modified_time
=
1
,
created_time
=
1
)
db
.
session
.
add
(
ui_file
)
db
.
session
.
commit
()
DatabaseService
.
commit
()
# create a fake old file, to be deleted on the next GET request
FileService
.
create_tmp_file
(
FileType
.
XML
,
"old"
)
args
:
dict
=
dict
(
type
=
FileType
.
XML
,
id
=
Mocks
.
exercise
.
eid
,
solution_indices
=
[
0
])
...
...
@@ -280,14 +279,14 @@ class McTestCase(unittest.TestCase):
with
open
(
file_path
,
"w+"
)
as
f
:
f
.
write
(
file_content
)
ui_file
.
last_modified_time
=
datetime
.
utcnow
().
timestamp
()
db
.
session
.
commit
()
DatabaseService
.
commit
()
del
ui_file
response
=
Mocks
.
app_dict
[
self
.
class_name
].
client
.
get
(
TestingConfig
.
SERVER_URI_FILE
,
query_string
=
args
)
os
.
remove
(
file_path
)
self
.
assertEqual
(
response
.
data
.
decode
(
"utf-8"
),
file_content
)
# add the mapped exercise to the database
db
.
session
.
add
(
Mocks
.
exercise
)
db
.
session
.
commit
()
DatabaseService
.
commit
()
args
[
"type"
]
=
FileType
.
PDF
response
=
Mocks
.
app_dict
[
self
.
class_name
].
client
.
get
(
TestingConfig
.
SERVER_URI_FILE
,
query_string
=
args
)
# the PDFs are not deterministically reproducible because the creation date etc. is written into them
...
...
@@ -326,15 +325,15 @@ class McTestCase(unittest.TestCase):
response
:
Response
=
Mocks
.
app_dict
[
self
.
class_name
].
client
.
get
(
TestingConfig
.
SERVER_URI_H5P
,
query_string
=
args
)
self
.
assertEqual
(
response
.
status_code
,
404
)
db
.
session
.
add
(
Mocks
.
exercise
)
db
.
session
.
commit
()
DatabaseService
.
commit
()
response
=
Mocks
.
app_dict
[
self
.
class_name
].
client
.
get
(
TestingConfig
.
SERVER_URI_H5P
,
query_string
=
args
)
self
.
assertIn
(
Mocks
.
h5p_json_cloze
[
1
:
-
1
],
response
.
data
.
decode
(
"utf-8"
))
Mocks
.
exercise
.
exercise_type
=
ExerciseType
.
kwic
.
value
db
.
session
.
commit
()
DatabaseService
.
commit
()
response
=
Mocks
.
app_dict
[
self
.
class_name
].
client
.
get
(
TestingConfig
.
SERVER_URI_H5P
,
query_string
=
args
)
self
.
assertEqual
(
response
.
status_code
,
422
)
Mocks
.
exercise
.
exercise_type
=
ExerciseType
.
matching
.
value
db
.
session
.
commit
()
DatabaseService
.
commit
()
response
=
Mocks
.
app_dict
[
self
.
class_name
].
client
.
get
(
TestingConfig
.
SERVER_URI_H5P
,
query_string
=
args
)
self
.
assertIn
(
Mocks
.
h5p_json_matching
[
1
:
-
1
],
response
.
data
.
decode
(
"utf-8"
))
Mocks
.
exercise
.
exercise_type
=
ExerciseType
.
cloze
.
value
...
...
@@ -530,6 +529,20 @@ class McTestCase(unittest.TestCase):
Mocks
.
app_dict
[
self
.
class_name
].
app_context
.
push
()
db
.
session
.
query
(
Corpus
).
delete
()
def
test_commit
(
self
):
"""Commits the last action to the database and, if it fails, rolls back the current session."""
def
commit
():
raise
OperationalError
(
""
,
[],
""
)
with
patch
.
object
(
mcserver
.
app
.
services
.
databaseService
,
"db"
)
as
mock_db
:
mock_db
.
session
.
commit
.
side_effect
=
commit
with
self
.
assertRaises
(
OperationalError
):
McTestCase
.
add_corpus
(
Mocks
.
corpora
[
0
])
db
.
session
.
query
(
Corpus
).
delete
()
db
.
session
.
query
(
UpdateInfo
).
delete
()
session
.
make_transient
(
Mocks
.
corpora
[
0
])
def
test_create_app
(
self
):
"""Creates a new Flask application and configures it. Initializes the application and the database."""
with
patch
.
object
(
sys
,
"argv"
,
[
None
,
None
,
Config
.
FLASK_MIGRATE
]):
...
...
@@ -554,7 +567,7 @@ class McTestCase(unittest.TestCase):
ui_cts
:
UpdateInfo
=
UpdateInfo
.
from_dict
(
resource_type
=
ResourceType
.
cts_data
.
name
,
last_modified_time
=
datetime
.
utcnow
().
timestamp
(),
created_time
=
1
)
db
.
session
.
add
(
ui_cts
)
db
.
session
.
commit
()
DatabaseService
.
commit
()
csm_process
:
Process
with
patch
.
object
(
sys
,
'argv'
,
Mocks
.
test_args
):
os
.
environ
[
Config
.
COVERAGE_ENVIRONMENT_VARIABLE
]
=
Config
.
COVERAGE_CONFIGURATION_FILE_NAME
...
...
@@ -573,7 +586,7 @@ class McTestCase(unittest.TestCase):
ui_exercises
:
UpdateInfo
=
UpdateInfo
.
from_dict
(
resource_type
=
ResourceType
.
exercise_list
.
name
,
last_modified_time
=
1
,
created_time
=
1
)
db
.
session
.
add
(
ui_exercises
)
db
.
session
.
commit
()
DatabaseService
.
commit
()
exercise_expected
:
Exercise
=
Mocks
.
exercise
exercise
:
Exercise
=
map_exercise_data_to_database
(
solutions
=
[
Solution
.
from_dict
(
x
)
for
x
in
json
.
loads
(
exercise_expected
.
solutions
)],
...
...
@@ -609,7 +622,7 @@ class McTestCase(unittest.TestCase):
self
.
assertEqual
(
len
(
CorpusService
.
existing_corpora
),
1
)
ec
:
Corpus
=
CorpusService
.
existing_corpora
[
0
]
ec
.
title
=
""
db
.
session
.
commit
()
DatabaseService
.
commit
()
McTestCase
.
add_corpus
(
CorpusMC
.
from_dict
(
source_urn
=
"123"
))
cls
:
List
[
CitationLevel
]
=
[
ec
.
citation_level_1
,
ec
.
citation_level_2
,
ec
.
citation_level_3
]
CorpusService
.
update_corpus
(
ec
.
title
,
ec
.
source_urn
,
ec
.
author
,
cls
,
ec
)
...
...
@@ -720,7 +733,7 @@ class CsmTestCase(unittest.TestCase):
ui_cts
:
UpdateInfo
=
UpdateInfo
.
from_dict
(
resource_type
=
ResourceType
.
cts_data
.
name
,
last_modified_time
=
1
,
created_time
=
1
)
db
.
session
.
add
(
ui_cts
)
db
.
session
.
commit
()
DatabaseService
.
commit
()
utc_now
:
datetime
=
datetime
.
utcnow
()
DatabaseService
.
check_corpus_list_age
(
Mocks
.
app_dict
[
self
.
class_name
].
app
)
ui_cts
:
UpdateInfo
=
db
.
session
.
query
(
UpdateInfo
).
filter_by
(
resource_type
=
ResourceType
.
cts_data
.
name
).
first
()
...
...
@@ -806,7 +819,7 @@ class CsmTestCase(unittest.TestCase):
ui_cts
:
UpdateInfo
=
UpdateInfo
.
from_dict
(
resource_type
=
ResourceType
.
cts_data
.
name
,
last_modified_time
=
1
,
created_time
=
1
)
db
.
session
.
add
(
ui_cts
)
db
.
session
.
commit
()
DatabaseService
.
commit
()
with
patch
.
object
(
CorpusService
,
'update_corpora'
)
as
update_mock
:
t
:
Thread
=
start_updater
(
Mocks
.
app_dict
[
self
.
class_name
].
app
)
self
.
assertIsInstance
(
t
,
Thread
)
...
...
@@ -866,7 +879,7 @@ class CommonTestCase(unittest.TestCase):
from
mcserver.app.api.vectorNetworkAPI
import
add_edges
w2v
:
Word2Vec
=
Word2Vec
([
x
.
split
()
for
x
in
Mocks
.
raw_text
.
split
(
". "
)],
min_count
=
1
,
sample
=
0
)
graph
:
Graph
=
Graph
()
add_edges
([
"fortis"
],
w2v
,
2
,
1
,
graph
)
add_edges
([
"fortis"
],
w2v
,
4
,
1
,
graph
)
self
.
assertGreater
(
len
(
graph
.
edges
),
1
)
def
test_add_urn_to_sentences
(
self
):
...
...
@@ -1180,7 +1193,7 @@ class CommonTestCase(unittest.TestCase):
ExerciseMC
.
from_dict
(
last_access_time
=
datetime
.
utcnow
().
timestamp
(),
urn
=
"urn"
,
solutions
=
json
.
dumps
([
Solution
().
to_dict
()]),
text_complexity
=
0
,
eid
=
"eid2"
)]
db
.
session
.
add_all
(
exercises
)
db
.
session
.
commit
()
DatabaseService
.
commit
()
with
patch
.
object
(
mcserver
.
app
.
services
.
textComplexityService
.
requests
,
"post"
,
return_value
=
MockResponse
(
Mocks
.
text_complexity_json_string
)):
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment