Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in / Register
Toggle navigation
Menu
Open sidebar
callidus
Machina Callida
Commits
90b6557c
Commit
90b6557c
authored
May 07, 2020
by
Konstantin Schulz
Browse files
added a script for testing code coverage locally
parent
763e0d22
Pipeline
#11492
passed with stages
in 3 minutes and 10 seconds
Changes
39
Pipelines
2
Hide whitespace changes
Inline
Side-by-side
.gitlab-ci.yml
View file @
90b6557c
...
...
@@ -23,7 +23,7 @@ ci_backend:
coverage
:
stage
:
coverage
script
:
-
./coverage.sh
-
./coverage
_ci
.sh
-
cat coverage.log
artifacts
:
paths
:
...
...
README.md
View file @
90b6557c
...
...
@@ -27,8 +27,10 @@ Alternatively, you can use `ssh root@localhost -p 8022 -o "UserKnownHostsFile /d
To snapshot a running container, use
`docker commit CONTAINER_ID`
. It returns a snapshot ID, which you can access via
`docker run -it SNAPSHOT_ID`
.
----------------------------------------------------------------
## Documentation
### Changelog
To update the changelog, use:
`git log --oneline --decorate > CHANGELOG`
## Testing
### Locally
To test your code locally, run
`./coverage_local.sh`
coverage.sh
→
coverage
_ci
.sh
View file @
90b6557c
File moved
coverage_local.sh
0 → 100755
View file @
90b6557c
docker-compose build
docker-compose run
--rm
--entrypoint
=
"npm run test-ci"
mc_frontend
>
ci_frontend.log
docker-compose run
--rm
mcserver bash
-c
"source ../venv/bin/activate && coverage run --rcfile=.coveragerc tests.py && coverage combine && coverage report -m"
>
ci_backend.log
./coverage_ci.sh
cat
coverage.log
docker-compose.yml
View file @
90b6557c
...
...
@@ -35,18 +35,6 @@ services:
tty
:
true
volumes
:
-
$PWD/mc_frontend/www:/home/mc/mc_frontend/www
nginx
:
command
:
nginx -g "daemon off;"
image
:
nginx:alpine
ports
:
-
"
8100:80"
restart
:
always
volumes
:
-
$PWD/mc_frontend/www:/usr/share/nginx/html
-
./mc_frontend/nginx.conf:/etc/nginx/nginx.conf
depends_on
:
-
mc_frontend
-
mcserver
mcserver
:
build
:
context
:
./mc_backend
...
...
@@ -63,5 +51,14 @@ services:
-
"
5000:5000"
restart
:
always
stdin_open
:
true
nginx
:
command
:
nginx -g "daemon off;"
image
:
nginx:alpine
ports
:
-
"
8100:80"
restart
:
always
volumes
:
-
$PWD/mc_frontend/www:/usr/share/nginx/html
-
./mc_frontend/nginx.conf:/etc/nginx/nginx.conf
volumes
:
db-data
:
mc_backend/.coveragerc
View file @
90b6557c
...
...
@@ -13,5 +13,6 @@ parallel = True
exclude_lines =
# Don't complain if non-runnable code isn't run:
if __name__ == .__main__.:
except ModuleNotFoundError:
ignore_errors = True
show_missing = True
mc_backend/app.py
View file @
90b6557c
...
...
@@ -4,4 +4,4 @@ from mcserver import get_app, get_cfg
app
:
Flask
=
get_app
()
if
__name__
==
"__main__"
:
app
.
run
(
host
=
get_cfg
().
HOST_IP
,
port
=
get_cfg
().
HOST_PORT
,
use_reloader
=
False
)
app
.
run
(
host
=
get_cfg
().
HOST_IP
_MCSERVER
,
port
=
get_cfg
().
HOST_PORT
,
use_reloader
=
False
)
mc_backend/csm/__init__.py
View file @
90b6557c
...
...
@@ -20,7 +20,7 @@ def get_cfg() -> Type[Config]:
def
run_app
()
->
None
:
cfg
:
Type
[
Config
]
=
get_cfg
()
get_app
().
run
(
host
=
cfg
.
HOST_IP
,
port
=
cfg
.
CORPUS_STORAGE_MANAGER_PORT
,
use_reloader
=
False
)
get_app
().
run
(
host
=
cfg
.
HOST_IP
_CSM
,
port
=
cfg
.
CORPUS_STORAGE_MANAGER_PORT
,
use_reloader
=
False
)
if
__name__
==
"__main__"
:
...
...
mc_backend/csm/__main__.py
View file @
90b6557c
from
csm
import
get_app
,
get_cfg
get_app
().
run
(
host
=
get_cfg
().
HOST_IP
,
port
=
get_cfg
().
HOST_PORT
,
use_reloader
=
False
)
get_app
().
run
(
host
=
get_cfg
().
HOST_IP
_CSM
,
port
=
get_cfg
().
HOST_PORT
,
use_reloader
=
False
)
mc_backend/csm/app/api/annisFindAPI.py
View file @
90b6557c
import
flask
from
flask_restful
import
Resource
,
reqparse
from
mcserver.app.services
import
NetworkService
,
CorpusService
,
AnnotationService
from
flask_restful
import
Resource
from
flask_restful.reqparse
import
RequestParser
from
mcserver.app.services
import
NetworkService
,
CorpusService
class
AnnisFindAPI
(
Resource
):
def
__init__
(
self
):
self
.
reqparse
=
reqparse
.
R
equest
P
arser
()
self
.
reqparse
:
RequestParser
=
NetworkService
.
base_r
equest
_p
arser
.
copy
()
self
.
reqparse
.
add_argument
(
"aql"
,
type
=
str
,
required
=
True
,
location
=
"form"
,
help
=
"No AQL provided"
)
self
.
reqparse
.
add_argument
(
"urn"
,
type
=
str
,
required
=
True
,
default
=
""
,
location
=
"form"
,
help
=
"No URN provided"
)
super
(
AnnisFindAPI
,
self
).
__init__
()
...
...
mc_backend/csm/app/api/corpusStorageManagerAPI.py
View file @
90b6557c
...
...
@@ -4,7 +4,8 @@ from typing import Dict, List
import
flask
from
conllu
import
TokenList
from
flask_restful
import
Resource
,
reqparse
,
abort
from
flask_restful
import
Resource
,
abort
from
flask_restful.reqparse
import
RequestParser
from
mcserver.app.models
import
ExerciseType
,
Phenomenon
,
AnnisResponse
from
mcserver.app.services
import
CorpusService
,
NetworkService
...
...
@@ -16,7 +17,7 @@ class CorpusStorageManagerAPI(Resource):
It manages the database and everything corpus-related."""
def
__init__
(
self
):
self
.
reqparse
=
reqparse
.
R
equest
P
arser
()
self
.
reqparse
:
RequestParser
=
NetworkService
.
base_r
equest
_p
arser
.
copy
()
self
.
reqparse
.
add_argument
(
"title"
,
type
=
str
,
required
=
True
,
location
=
"data"
,
help
=
"No title provided"
)
self
.
reqparse
.
add_argument
(
"annotations"
,
required
=
True
,
location
=
"data"
,
help
=
"No annotations provided"
)
...
...
mc_backend/csm/app/api/frequencyAPI.py
View file @
90b6557c
from
typing
import
List
,
Dict
,
Set
import
flask
from
flask_restful
import
Resource
,
reqparse
from
flask_restful
import
Resource
from
flask_restful.reqparse
import
RequestParser
from
mcserver.app.models
import
FrequencyAnalysis
,
Phenomenon
from
mcserver.app.services
import
NetworkService
,
CorpusService
,
AnnotationService
class
FrequencyAPI
(
Resource
):
def
__init__
(
self
):
self
.
reqparse
=
reqparse
.
R
equest
P
arser
()
self
.
reqparse
:
RequestParser
=
NetworkService
.
base_r
equest
_p
arser
.
copy
()
self
.
reqparse
.
add_argument
(
"urn"
,
type
=
str
,
required
=
True
,
default
=
""
,
location
=
"form"
,
help
=
"No URN provided"
)
super
(
FrequencyAPI
,
self
).
__init__
()
...
...
mc_backend/csm/app/api/subgraphAPI.py
View file @
90b6557c
...
...
@@ -2,7 +2,9 @@ import json
from
typing
import
Dict
,
List
import
flask
from
flask_restful
import
Resource
,
reqparse
from
flask_restful
import
Resource
from
flask_restful.reqparse
import
RequestParser
from
mcserver.app.models
import
ExerciseData
,
GraphData
,
Solution
,
SolutionElement
,
AnnisResponse
from
mcserver.app.services
import
CorpusService
,
AnnotationService
,
NetworkService
...
...
@@ -10,7 +12,7 @@ from mcserver.app.services import CorpusService, AnnotationService, NetworkServi
class
SubgraphAPI
(
Resource
):
def
__init__
(
self
):
self
.
reqparse
=
reqparse
.
R
equest
P
arser
()
self
.
reqparse
:
RequestParser
=
NetworkService
.
base_r
equest
_p
arser
.
copy
()
self
.
reqparse
.
add_argument
(
"aqls"
,
required
=
False
,
location
=
"data"
,
help
=
"No AQLs provided"
,
action
=
"append"
)
self
.
reqparse
.
add_argument
(
"ctx_left"
,
type
=
str
,
required
=
False
,
default
=
""
,
location
=
"data"
,
help
=
"No left context provided"
)
...
...
mc_backend/csm/app/api/textcomplexityAPI.py
View file @
90b6557c
import
rapidjson
as
json
import
flask
from
flask_restful
import
Resource
,
reqparse
from
flask_restful
import
Resource
from
flask_restful.reqparse
import
RequestParser
from
mcserver.app.models
import
AnnisResponse
,
GraphData
,
TextComplexity
from
mcserver.app.services
import
NetworkService
,
CorpusService
,
TextComplexityService
...
...
@@ -11,7 +12,7 @@ class TextComplexityAPI(Resource):
"""The Text Complexity API resource. It gives users measures for text complexity for a given text."""
def
__init__
(
self
):
self
.
reqparse
=
reqparse
.
R
equest
P
arser
()
self
.
reqparse
:
RequestParser
=
NetworkService
.
base_r
equest
_p
arser
.
copy
()
self
.
reqparse
.
add_argument
(
'urn'
,
type
=
str
,
location
=
"data"
,
required
=
True
,
help
=
'No URN provided'
)
self
.
reqparse
.
add_argument
(
'measure'
,
type
=
str
,
location
=
"data"
,
required
=
True
,
help
=
'No MEASURE provided'
)
self
.
reqparse
.
add_argument
(
'annis_response'
,
type
=
dict
,
location
=
"data"
,
required
=
False
,
...
...
mc_backend/csm/gunicorn_config.py
View file @
90b6557c
"""Configuration for the gunicorn server"""
from
mcserver
import
Config
bind
=
"{0}:{1}"
.
format
(
Config
.
HOST_IP
,
Config
.
CORPUS_STORAGE_MANAGER_PORT
)
bind
=
"{0}:{1}"
.
format
(
Config
.
HOST_IP
_CSM
,
Config
.
CORPUS_STORAGE_MANAGER_PORT
)
debug
=
False
reload
=
True
timeout
=
3600
...
...
mc_backend/mcserver/__init__.py
View file @
90b6557c
...
...
@@ -20,4 +20,4 @@ def get_cfg() -> Type[Config]:
if
__name__
==
"__main__"
:
# reloader has to be disabled because of a bug with Flask and multiprocessing
get_app
().
run
(
host
=
get_cfg
().
HOST_IP
,
port
=
get_cfg
().
HOST_PORT
,
use_reloader
=
False
)
get_app
().
run
(
host
=
get_cfg
().
HOST_IP
_MCSERVER
,
port
=
get_cfg
().
HOST_PORT
,
use_reloader
=
False
)
mc_backend/mcserver/__main__.py
View file @
90b6557c
from
mcserver
import
get_app
,
get_cfg
get_app
().
run
(
host
=
get_cfg
().
HOST_IP
,
port
=
get_cfg
().
HOST_PORT
,
use_reloader
=
False
)
get_app
().
run
(
host
=
get_cfg
().
HOST_IP
_MCSERVER
,
port
=
get_cfg
().
HOST_PORT
,
use_reloader
=
False
)
mc_backend/mcserver/app/__init__.py
View file @
90b6557c
...
...
@@ -5,7 +5,9 @@ from logging.handlers import RotatingFileHandler
from
threading
import
Thread
from
time
import
strftime
from
typing
import
Type
from
flask
import
Flask
,
got_request_exception
,
request
,
Response
import
flask
from
flask
import
Flask
,
got_request_exception
,
request
,
Response
,
send_from_directory
from
flask_cors
import
CORS
from
flask_migrate
import
Migrate
from
flask_sqlalchemy
import
SQLAlchemy
...
...
@@ -59,6 +61,12 @@ def init_app_common(cfg: Type[Config] = Config, is_csm: bool = False) -> Flask:
request
.
full_path
,
response
.
status
)
return
response
@
app
.
route
(
Config
.
SERVER_URI_FAVICON
)
def
get_favicon
():
"""Sends the favicon to browsers, which is used, e.g., in the tabs as a symbol for our application."""
mime_type
:
str
=
'image/vnd.microsoft.icon'
return
send_from_directory
(
Config
.
ASSETS_DIRECTORY
,
Config
.
FAVICON_FILE_NAME
,
mimetype
=
mime_type
)
@
app
.
teardown_appcontext
def
shutdown_session
(
exception
=
None
):
""" Shuts down the session when the application exits. (maybe also after every request ???) """
...
...
@@ -100,7 +108,7 @@ def log_exception(sender_app: Flask, exception, **extra):
**extra -- any additional arguments
"""
# TODO: RETURN ERROR IN JSON FORMAT
sender_app
.
logger
.
exception
(
"ERROR"
)
sender_app
.
logger
.
exception
(
f
"ERROR
for
{
flask
.
request
.
url
}
"
)
def
start_updater
(
app
:
Flask
)
->
Thread
:
...
...
mc_backend/mcserver/app/api/corpusAPI.py
View file @
90b6557c
"""The corpus API. Add it to your REST API to provide users with metadata about specific texts."""
from
flask_restful
import
Resource
,
reqparse
,
abort
,
marshal
from
flask_restful
import
Resource
,
abort
,
marshal
from
flask_restful.reqparse
import
RequestParser
from
mcserver.app
import
db
from
mcserver.app.models
import
Corpus
,
corpus_fields
from
mcserver.app.services
import
NetworkService
class
CorpusAPI
(
Resource
):
...
...
@@ -10,7 +12,7 @@ class CorpusAPI(Resource):
def
__init__
(
self
):
"""Initialize possible arguments for calls to the corpus REST API."""
self
.
reqparse
=
reqparse
.
R
equest
P
arser
()
self
.
reqparse
:
RequestParser
=
NetworkService
.
base_r
equest
_p
arser
.
copy
()
self
.
reqparse
.
add_argument
(
"title"
,
type
=
str
,
required
=
False
,
help
=
"No title provided"
)
self
.
reqparse
.
add_argument
(
"author"
,
type
=
str
,
required
=
False
,
help
=
"No author provided"
)
self
.
reqparse
.
add_argument
(
"source_urn"
,
type
=
str
,
required
=
False
,
help
=
"No source URN provided"
)
...
...
mc_backend/mcserver/app/api/corpusListAPI.py
View file @
90b6557c
...
...
@@ -2,12 +2,13 @@
from
datetime
import
datetime
from
flask
import
jsonify
from
flask_restful
import
Resource
,
reqparse
,
marshal
from
flask_restful
import
Resource
,
marshal
from
flask_restful.reqparse
import
RequestParser
from
sqlalchemy.exc
import
OperationalError
,
InvalidRequestError
from
mcserver.app
import
db
from
mcserver.app.models
import
UpdateInfo
,
ResourceType
,
Corpus
,
corpus_fields
from
mcserver.app.services
import
CorpusService
from
mcserver.app.services
import
CorpusService
,
NetworkService
class
CorpusListAPI
(
Resource
):
...
...
@@ -15,7 +16,7 @@ class CorpusListAPI(Resource):
def
__init__
(
self
):
"""Initialize possible arguments for calls to the corpus list REST API."""
self
.
reqparse
=
reqparse
.
R
equest
P
arser
()
self
.
reqparse
:
RequestParser
=
NetworkService
.
base_r
equest
_p
arser
.
copy
()
self
.
reqparse
.
add_argument
(
"last_update_time"
,
type
=
int
,
required
=
True
,
help
=
"No milliseconds time for last update provided"
)
super
(
CorpusListAPI
,
self
).
__init__
()
...
...
Prev
1
2
Next
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment