From 3d1c53e9e5d23bcc912d02c0a62a5e7270ba3d31 Mon Sep 17 00:00:00 2001 From: Ralf Grubenmann Date: Fri, 29 Nov 2019 13:03:57 +0100 Subject: [PATCH 01/11] SHACL Validation (#767) * Adds SHACL validation to renku doctor * Adds renku log tests * Adds --strict option to renku log for shape validation --- MANIFEST.in | 3 +- renku/cli/log.py | 28 +- renku/core/commands/checks/__init__.py | 3 + renku/core/commands/checks/validate_shacl.py | 114 +++ renku/core/commands/dataset.py | 2 +- renku/core/commands/format/graph.py | 95 ++- renku/core/compat.py | 26 +- renku/core/errors.py | 4 + renku/core/models/datasets.py | 18 +- renku/core/models/entities.py | 4 +- renku/core/models/jsonld.py | 10 +- renku/core/models/provenance/activities.py | 13 +- renku/core/models/provenance/agents.py | 7 +- renku/core/utils/shacl.py | 44 ++ renku/data/shacl_shape.json | 780 +++++++++++++++++++ setup.py | 1 + tests/cli/test_integration_datasets.py | 39 +- tests/cli/test_log.py | 69 ++ tests/cli/test_update.py | 8 +- tests/core/models/test_shacl_schema.py | 106 +++ tests/fixtures/force_dataset_shacl.json | 32 + tests/fixtures/force_datasetfile_shacl.json | 32 + tests/fixtures/force_datasettag_shacl.json | 32 + tests/fixtures/force_project_shacl.json | 32 + 24 files changed, 1430 insertions(+), 72 deletions(-) create mode 100644 renku/core/commands/checks/validate_shacl.py create mode 100644 renku/core/utils/shacl.py create mode 100644 renku/data/shacl_shape.json create mode 100644 tests/cli/test_log.py create mode 100644 tests/core/models/test_shacl_schema.py create mode 100644 tests/fixtures/force_dataset_shacl.json create mode 100644 tests/fixtures/force_datasetfile_shacl.json create mode 100644 tests/fixtures/force_datasettag_shacl.json create mode 100644 tests/fixtures/force_project_shacl.json diff --git a/MANIFEST.in b/MANIFEST.in index ab3e9b5e3c..8c9e058fa7 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -56,6 +56,7 @@ recursive-include renku *.html recursive-include renku *.sh recursive-include renku *.txt recursive-include renku *.yml +recursive-include renku *.json recursive-include renku Dockerfile -recursive-include tests *.py *.gz *.yml +recursive-include tests *.py *.gz *.yml *.json prune .github diff --git a/renku/cli/log.py b/renku/cli/log.py index fa4fc83dfe..1873314fe2 100644 --- a/renku/cli/log.py +++ b/renku/cli/log.py @@ -52,6 +52,15 @@ * `ascii` * `dot` +* `dot-full` +* `dot-landscape` +* `dot-full-landscape` +* `dot-debug` +* `json-ld` +* `json-ld-graph` +* `Makefile` +* `nt` +* `rdf` You can generate a PNG of the full history of all files in the repository using the :program:`dot` program. @@ -62,6 +71,15 @@ $ renku log --format dot $FILES | dot -Tpng > /tmp/graph.png $ open /tmp/graph.png +Output validation +~~~~~~~~~~~~~~~~~ + +The ``--strict`` option forces the output to be validated against the Renku +SHACL schema, causing the command to fail if the generated output is not +valid, as well as printing detailed information on all the issues found. +The ``--strict`` option is only supported for the ``jsonld``, ``rdf`` and +``nt`` output formats. + """ import click @@ -86,9 +104,15 @@ default=False, help='Display commands without output files.' ) +@click.option( + '--strict', + is_flag=True, + default=False, + help='Validate triples before output.' +) @click.argument('paths', type=click.Path(exists=True), nargs=-1) @pass_local_client -def log(client, revision, format, no_output, paths): +def log(client, revision, format, no_output, strict, paths): """Show logs for a file.""" graph = Graph(client) if not paths: @@ -108,4 +132,4 @@ def log(client, revision, format, no_output, paths): # NOTE shall we warn when "not no_output and not paths"? graph.build(paths=paths, revision=revision, can_be_cwl=no_output) - FORMATS[format](graph) + FORMATS[format](graph, strict=strict) diff --git a/renku/core/commands/checks/__init__.py b/renku/core/commands/checks/__init__.py index 6025f1fd4c..48ab9788b8 100644 --- a/renku/core/commands/checks/__init__.py +++ b/renku/core/commands/checks/__init__.py @@ -19,6 +19,7 @@ from .migration import check_dataset_metadata, check_missing_files from .references import check_missing_references +from .validate_shacl import check_project_structure, check_datasets_structure # Checks will be executed in the order as they are listed in __all__. # They are mostly used in ``doctor`` command to inspect broken things. @@ -26,4 +27,6 @@ 'check_dataset_metadata', 'check_missing_files', 'check_missing_references', + 'check_project_structure', + 'check_datasets_structure', ) diff --git a/renku/core/commands/checks/validate_shacl.py b/renku/core/commands/checks/validate_shacl.py new file mode 100644 index 0000000000..ae7c212a17 --- /dev/null +++ b/renku/core/commands/checks/validate_shacl.py @@ -0,0 +1,114 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Check KG structure using SHACL.""" +import yaml +from rdflib.namespace import Namespace +from rdflib.term import BNode + +from renku.core.commands.echo import WARNING +from renku.core.compat import pyld +from renku.core.models.jsonld import NoDatesSafeLoader +from renku.core.utils.shacl import validate_graph + + +def _shacl_graph_to_string(graph): + """Converts a shacl validation graph into human readable format.""" + sh = Namespace('http://www.w3.org/ns/shacl#') + + problems = [] + + for _, result in graph.subject_objects(sh.result): + path = graph.value(result, sh.resultPath) + res = graph.value(result, sh.resultMessage) + + if res: + message = '{0}: {1}'.format(path, res) + else: + kind = graph.value(result, sh.sourceConstraintComponent) + focusNode = graph.value(result, sh.focusNode) + + if isinstance(focusNode, BNode): + focusNode = '' + + message = '{0}: Type: {1}, Node ID: {2}'.format( + path, kind, focusNode + ) + + problems.append(message) + + return '\n\t'.join(problems) + + +def check_project_structure(client): + """Validate project metadata against SHACL.""" + project_path = client.renku_metadata_path + + conform, graph, t = check_shacl_structure(project_path) + + if conform: + return True, None + + problems = '{0}Invalid structure of project metadata\n\t{1}'.format( + WARNING, _shacl_graph_to_string(graph) + ) + + return False, problems + + +def check_datasets_structure(client): + """Validate dataset metadata against SHACL.""" + ok = True + + problems = ['{0}Invalid structure of dataset metadata'.format(WARNING)] + + for path in client.renku_datasets_path.rglob(client.METADATA): + try: + conform, graph, t = check_shacl_structure(path) + except (Exception, BaseException) as e: + problems.append('Couldn\'t validate {0}: {1}\n\n'.format(path, e)) + continue + + if conform: + continue + + ok = False + + problems.append( + '{0}\n\t{1}\n'.format(path, _shacl_graph_to_string(graph)) + ) + + if ok: + return True, None + + return False, '\n'.join(problems) + + +def check_shacl_structure(path): + """Validates all metadata aginst the SHACL schema.""" + with path.open(mode='r') as fp: + source = yaml.load(fp, Loader=NoDatesSafeLoader) or {} + + rdf = pyld.jsonld.to_rdf( + source, + options={ + 'format': 'application/n-quads', + 'produceGeneralizedRdf': True + } + ) + + return validate_graph(rdf) diff --git a/renku/core/commands/dataset.py b/renku/core/commands/dataset.py index 288a8983bf..6604b2838a 100644 --- a/renku/core/commands/dataset.py +++ b/renku/core/commands/dataset.py @@ -567,7 +567,7 @@ def update_datasets( file_.dataset = dataset possible_updates.append(file_) - unique_remotes.add(file_.based_on['url']) + unique_remotes.add(file_.based_on.url) if ref and len(unique_remotes) > 1: raise ParameterError( diff --git a/renku/core/commands/format/graph.py b/renku/core/commands/format/graph.py index 4dc3eec0f1..ef4c19f1d5 100644 --- a/renku/core/commands/format/graph.py +++ b/renku/core/commands/format/graph.py @@ -21,12 +21,18 @@ import click +from renku.core.errors import SHACLValidationError +from renku.core.utils.shacl import validate_graph -def ascii(graph): + +def ascii(graph, strict=False): """Format graph as an ASCII art.""" from ..ascii import DAG from ..echo import echo_via_pager + if strict: + raise SHACLValidationError('--strict not supported for json-ld-graph') + echo_via_pager(str(DAG(graph))) @@ -34,30 +40,39 @@ def _jsonld(graph, format, *args, **kwargs): """Return formatted graph in JSON-LD ``format`` function.""" import json - from pyld import jsonld + from renku.core.compat import pyld from renku.core.models.jsonld import asjsonld - output = getattr(jsonld, format)([ + output = getattr(pyld.jsonld, format)([ asjsonld(action) for action in graph.activities.values() ]) return json.dumps(output, indent=2) -def dot(graph, simple=True, debug=False, landscape=False): - """Format graph as a dot file.""" - import sys - +def _conjunctive_graph(graph): + """Convert a renku ``Graph`` to an rdflib ``ConjunctiveGraph``.""" from rdflib import ConjunctiveGraph from rdflib.plugin import register, Parser - from rdflib.tools.rdf2dot import rdf2dot register('json-ld', Parser, 'rdflib_jsonld.parser', 'JsonLDParser') - g = ConjunctiveGraph().parse( + return ConjunctiveGraph().parse( data=_jsonld(graph, 'expand'), format='json-ld', ) + +def dot(graph, simple=True, debug=False, landscape=False, strict=False): + """Format graph as a dot file.""" + import sys + + from rdflib.tools.rdf2dot import rdf2dot + + if strict: + raise SHACLValidationError('--strict not supported for json-ld-graph') + + g = _conjunctive_graph(graph) + g.bind('prov', 'http://www.w3.org/ns/prov#') g.bind('foaf', 'http://xmlns.com/foaf/0.1/') g.bind('wfdesc', 'http://purl.org/wf4ever/wfdesc#') @@ -92,7 +107,7 @@ def _rdf2dot_simple(g, stream): import re path_re = re.compile( - r'file:///(?P[a-zA-Z]+)/' + r'(?Pfile://|https://\w+/\w+/){0,1}(?P[a-zA-Z]+)/' r'(?P\w+)' r'(?P.+)?' ) @@ -293,10 +308,13 @@ def color(p): stream.write('}\n') -def makefile(graph): +def makefile(graph, strict=False): """Format graph as Makefile.""" from renku.core.models.provenance.activities import ProcessRun, WorkflowRun + if strict: + raise SHACLValidationError('--strict not supported for json-ld-graph') + for activity in graph.activities.values(): if not isinstance(activity, ProcessRun): continue @@ -316,44 +334,53 @@ def makefile(graph): ) -def jsonld(graph): +def jsonld(graph, strict=False): """Format graph as JSON-LD file.""" - click.echo(_jsonld(graph, 'expand')) + ld = _jsonld(graph, 'expand') + + if strict: + r, _, t = validate_graph(ld, format='json-ld') + + if not r: + raise SHACLValidationError( + "{}\nCouldn't get log: Invalid Knowledge Graph data".format(t) + ) + click.echo(ld) -def jsonld_graph(graph): +def jsonld_graph(graph, strict=False): """Format graph as JSON-LD graph file.""" + if strict: + raise SHACLValidationError('--strict not supported for json-ld-graph') click.echo(_jsonld(graph, 'flatten')) -def nt(graph): +def nt(graph, strict=False): """Format graph as n-tuples.""" - from rdflib import ConjunctiveGraph - from rdflib.plugin import register, Parser + nt = _conjunctive_graph(graph).serialize(format='nt') + if strict: + r, _, t = validate_graph(nt, format='nt') - register('json-ld', Parser, 'rdflib_jsonld.parser', 'JsonLDParser') + if not r: + raise SHACLValidationError( + "{}\nCouldn't get log: Invalid Knowledge Graph data".format(t) + ) - click.echo( - ConjunctiveGraph().parse( - data=_jsonld(graph, 'expand'), - format='json-ld', - ).serialize(format='nt') - ) + click.echo(nt) -def rdf(graph): +def rdf(graph, strict=False): """Output the graph as RDF.""" - from rdflib import ConjunctiveGraph - from rdflib.plugin import register, Parser + xml = _conjunctive_graph(graph).serialize(format='application/rdf+xml') + if strict: + r, _, t = validate_graph(xml, format='xml') - register('json-ld', Parser, 'rdflib_jsonld.parser', 'JsonLDParser') + if not r: + raise SHACLValidationError( + "{}\nCouldn't get log: Invalid Knowledge Graph data".format(t) + ) - click.echo( - ConjunctiveGraph().parse( - data=_jsonld(graph, 'expand'), - format='json-ld', - ).serialize(format='application/rdf+xml') - ) + click.echo(xml) FORMATS = { diff --git a/renku/core/compat.py b/renku/core/compat.py index cfbbda9161..8bc6ed24d4 100644 --- a/renku/core/compat.py +++ b/renku/core/compat.py @@ -18,10 +18,13 @@ """Compatibility layer for different Python versions.""" import contextlib +import json import os import sys from pathlib import Path +import pyld + if sys.version_info < (3, 6): original_resolve = Path.resolve @@ -63,4 +66,25 @@ def __exit__(self, *excinfo): except NameError: # pragma: no cover FileNotFoundError = IOError -__all__ = ('FileNotFoundError', 'Path', 'contextlib') + +class PatchedActiveContextCache(pyld.jsonld.ActiveContextCache): + """Pyld context cache without issue of missing contexts.""" + + def set(self, active_ctx, local_ctx, result): + if len(self.order) == self.size: + entry = self.order.popleft() + if sum( + e['activeCtx'] == entry['activeCtx'] and + e['localCtx'] == entry['localCtx'] for e in self.order + ) == 0: + # only delete from cache if it doesn't exist in context deque + del self.cache[entry['activeCtx']][entry['localCtx']] + key1 = json.dumps(active_ctx) + key2 = json.dumps(local_ctx) + self.order.append({'activeCtx': key1, 'localCtx': key2}) + self.cache.setdefault(key1, {})[key2] = json.loads(json.dumps(result)) + + +pyld.jsonld._cache = {'activeCtx': PatchedActiveContextCache()} + +__all__ = ('FileNotFoundError', 'Path', 'contextlib', 'pyld') diff --git a/renku/core/errors.py b/renku/core/errors.py index 473ab9701c..92cfe46b36 100644 --- a/renku/core/errors.py +++ b/renku/core/errors.py @@ -373,3 +373,7 @@ class UrlSchemeNotSupported(RenkuException): class OperationError(RenkuException): """Raised when an operation at runtime raises an error.""" + + +class SHACLValidationError(RenkuException): + """Raises when SHACL validation of the graph fails.""" diff --git a/renku/core/models/datasets.py b/renku/core/models/datasets.py index e178440f3d..22295b5e6c 100644 --- a/renku/core/models/datasets.py +++ b/renku/core/models/datasets.py @@ -127,7 +127,7 @@ def _now(self): @_id.default def default_id(self): """Define default value for id field.""" - return '{0}@{1}'.format(self.name, self.commit) + return '_:{0}@{1}'.format(self.name, self.commit) @jsonld.s( @@ -150,6 +150,12 @@ def convert_filename_path(p): return Path(p).name +def convert_based_on(v): + """Convert based_on to DatasetFile.""" + if v: + return DatasetFile.from_jsonld(v) + + @jsonld.s( type='schema:DigitalDocument', slots=True, @@ -179,7 +185,10 @@ class DatasetFile(Entity, CreatorMixin): url = jsonld.ib(default=None, context='schema:url', kw_only=True) based_on = jsonld.ib( - default=None, context='schema:isBasedOn', kw_only=True + default=None, + context='schema:isBasedOn', + kw_only=True, + converter=convert_based_on ) @added.default @@ -213,6 +222,11 @@ def __attrs_post_init__(self): if not self.name: self.name = self.filename + parsed_id = urllib.parse.urlparse(self._id) + + if not parsed_id.scheme: + self._id = 'file://{}'.format(self._id) + def _convert_dataset_files(value): """Convert dataset files.""" diff --git a/renku/core/models/entities.py b/renku/core/models/entities.py index 76fc9c8ffb..ebff9c4407 100644 --- a/renku/core/models/entities.py +++ b/renku/core/models/entities.py @@ -62,7 +62,9 @@ def default_id(self): hexsha = self.commit.hexsha else: hexsha = 'UNCOMMITTED' - return 'blob/{hexsha}/{self.path}'.format(hexsha=hexsha, self=self) + return 'file://blob/{hexsha}/{self.path}'.format( + hexsha=hexsha, self=self + ) @_label.default def default_label(self): diff --git a/renku/core/models/jsonld.py b/renku/core/models/jsonld.py index 8f2ac38413..5e2a9adbd8 100644 --- a/renku/core/models/jsonld.py +++ b/renku/core/models/jsonld.py @@ -30,8 +30,8 @@ from attr._compat import iteritems from attr._funcs import has from attr._make import Factory, fields -from pyld import jsonld as ld +from renku.core.compat import pyld from renku.core.models.locals import ReferenceMixin, with_reference from renku.core.models.migrations import JSONLD_MIGRATIONS @@ -149,7 +149,7 @@ def wrap(cls): # Register class for given JSON-LD @type try: - type_ = ld.expand({ + type_ = pyld.jsonld.expand({ '@type': jsonld_cls._jsonld_type, '@context': context })[0]['@type'] @@ -473,10 +473,10 @@ def from_jsonld( if cls._jsonld_translate: # perform the translation - data = ld.compact(data, cls._jsonld_translate) + data = pyld.jsonld.compact(data, cls._jsonld_translate) # compact using the class json-ld context data.pop('@context', None) - data = ld.compact(data, cls._jsonld_context) + data = pyld.jsonld.compact(data, cls._jsonld_context) data.setdefault('@context', cls._jsonld_context) @@ -504,7 +504,7 @@ def from_jsonld( data['@context'] = {'@base': data['@context']} data['@context'].update(cls._jsonld_context) try: - compacted = ld.compact(data, cls._jsonld_context) + compacted = pyld.jsonld.compact(data, cls._jsonld_context) except Exception: compacted = data else: diff --git a/renku/core/models/provenance/activities.py b/renku/core/models/provenance/activities.py index 1ac7969a41..b658adc0a6 100644 --- a/renku/core/models/provenance/activities.py +++ b/renku/core/models/provenance/activities.py @@ -18,9 +18,10 @@ """Represent a Git commit.""" import os +import urllib import uuid from collections import OrderedDict -from pathlib import Path +from pathlib import Path, posixpath import attr from git import NULL_TREE @@ -217,7 +218,15 @@ def paths(self): @classmethod def generate_id(cls, commit): """Calculate action ID.""" - return 'commit/{commit.hexsha}'.format(commit=commit) + host = os.environ.get('RENKU_DOMAIN') or 'localhost' + + # always set the id by the identifier + return urllib.parse.urljoin( + 'https://{host}'.format(host=host), + posixpath.join( + '/activities', 'commit/{commit.hexsha}'.format(commit=commit) + ) + ) @_id.default def default_id(self): diff --git a/renku/core/models/provenance/agents.py b/renku/core/models/provenance/agents.py index 67f429aef9..7e731b1c5a 100644 --- a/renku/core/models/provenance/agents.py +++ b/renku/core/models/provenance/agents.py @@ -58,9 +58,14 @@ class Person: @_id.default def default_id(self): """Set the default id.""" + import string if self.email: return 'mailto:{email}'.format(email=self.email) - return '_:{}'.format(''.join(self.name.lower().split())) + + # prep name to be a valid ntuple string + name = self.name.translate(str.maketrans('', '', string.punctuation)) + name = ''.join(filter(lambda x: x in string.printable, name)) + return '_:{}'.format(''.join(name.lower().split())) @email.validator def check_email(self, attribute, value): diff --git a/renku/core/utils/shacl.py b/renku/core/utils/shacl.py new file mode 100644 index 0000000000..71e2a15eba --- /dev/null +++ b/renku/core/utils/shacl.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2018-2019- Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""JSON-LD SHACL validations.""" + +from pkg_resources import resource_string +from pyshacl import validate + + +def validate_graph(graph, shacl_path=None, format='nquads'): + """Validate the current graph with a SHACL schema. + + Uses default schema if not supplied. + """ + if shacl_path: + with open(shacl_path, 'r', encoding='utf-8') as f: + shacl = f.read() + else: + shacl = resource_string('renku', 'data/shacl_shape.json') + + return validate( + graph, + shacl_graph=shacl, + inference='rdfs', + meta_shacl=True, + debug=False, + data_graph_format=format, + shacl_graph_format='json-ld', + advanced=True + ) diff --git a/renku/data/shacl_shape.json b/renku/data/shacl_shape.json new file mode 100644 index 0000000000..31d53f2277 --- /dev/null +++ b/renku/data/shacl_shape.json @@ -0,0 +1,780 @@ +{ + "@context": { + "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + "rdfs": "http://www.w3.org/2000/01/rdf-schema#", + "sh": "http://www.w3.org/ns/shacl#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "schema": "http://schema.org/", + "foaf": "http://xmlns.com/foaf/0.1/", + "prov": "http://www.w3.org/ns/prov#", + "wfprov": "http://purl.org/wf4ever/wfprov#", + "closed": { + "@id": "sh:closed", + "@type": "http://www.w3.org/2001/XMLSchema#boolean" + }, + "datatype": { + "@id": "sh:datatype", + "@type": "@id" + }, + "ignoredProperties": { + "@id": "sh:ignoredProperties", + "@container": "@list" + }, + "or": { + "@id": "sh:or", + "@container": "@list" + }, + "minCount": "sh:minCount", + "maxCount": "sh:maxCount", + "nodeKind": { + "@id": "sh:nodeKind", + "@type": "@id" + }, + "property": "sh:property", + "path": { + "@id": "sh:path", + "@type": "@id" + }, + "targetClass": { + "@id": "sh:targetClass", + "@type": "@id" + }, + "target": { + "@id": "sh:target", + "@type": "@id" + } + }, + "@graph": [ + { + "@id": "schema:", + "sh:declare": [ + { + "sh:prefix": [ + { + "@value": "schema" + } + ], + "sh:namespace": [ + { + "@value": "http://schema.org/", + "@type": "xsd:anyURI" + } + ] + } + ] + }, + { + "@id": "prov:", + "sh:declare": [ + { + "sh:prefix": [ + { + "@value": "prov" + } + ], + "sh:namespace": [ + { + "@value": "http://www.w3.org/ns/prov#", + "@type": "xsd:anyURI" + } + ] + } + ] + }, + { + "@id": "_:oldProjecShape", + "@type": "sh:NodeShape", + "targetClass": "foaf:Project", + "property": [ + { + "nodeKind": "sh:Literal", + "path": "ex:CheckOldProjectMetadata", + "minCount": 99999, + "maxCount": 99999, + "sh:message": "Project should be schema:Project, not foaf:Project" + } + ] + }, + { + "@id": "_:projectShape", + "@type": "sh:NodeShape", + "ignoredProperties": [ + { + "@id": "rdf:type" + } + ], + "closed": true, + "targetClass": "schema:Project", + "property": [ + { + "nodeKind": "sh:Literal", + "path": "schema:dateCreated", + "datatype": { + "@id": "xsd:string" + }, + "minCount": 1, + "maxCount": 1, + "sh:lessThanOrEquals": { + "@id": "schema:dateUpdated" + } + }, + { + "nodeKind": "sh:Literal", + "path": "schema:dateUpdated", + "datatype": { + "@id": "xsd:string" + }, + "minCount": 1, + "maxCount": 1 + }, + { + "nodeKind": "sh:Literal", + "path": "schema:schemaVersion", + "datatype": { + "@id": "xsd:string" + }, + "minCount": 1, + "maxCount": 1 + }, + { + "nodeKind": "sh:Literal", + "path": "schema:name", + "datatype": { + "@id": "xsd:string" + }, + "minCount": 1, + "maxCount": 1 + }, + { + "path": "schema:creator", + "sh:class":{ + "@id": "schema:Person" + }, + "minCount": 1 + } + ] + }, + { + "@id": "_:creatorShape", + "@type": "sh:NodeShape", + "ignoredProperties": [ + { + "@id": "rdf:type" + } + ], + "closed": true, + "target": [ + { + "@type": "sh:SPARQLTarget", + "sh:prefixes": [ + { + "@id": "schema:" + }, + { + "@id": "prov:" + } + ], + "sh:select": [ + { + "@value": "SELECT ?this\nWHERE {\n ?this a schema:Person .\n MINUS { ?this a prov:Person . }\n}\n" + } + ] + } + ], + "property": [ + { + "nodeKind": "sh:Literal", + "path": "schema:name", + "datatype": { + "@id": "xsd:string" + }, + "minCount": 1, + "maxCount": 1 + }, + { + "nodeKind": "sh:Literal", + "path": "schema:email", + "datatype": { + "@id": "xsd:string" + }, + "maxCount": 1 + }, + { + "nodeKind": "sh:Literal", + "path": "schema:alternateName", + "datatype": { + "@id": "xsd:string" + } + }, + { + "nodeKind": "sh:Literal", + "path": "schema:affiliation", + "datatype": { + "@id": "xsd:string" + } + } + ] + }, + { + "@id": "_:datasetShape", + "@type": "sh:NodeShape", + "ignoredProperties": [ + { + "@id": "rdf:type" + }, + { + "@id": "schema:license" + } + ], + "closed": true, + "target": [ + { + "@type": "sh:SPARQLTarget", + "sh:prefixes": [ + { + "@id": "schema:" + } + ], + "sh:select": [ + { + "@value": "SELECT ?this\nWHERE {\n ?this a schema:Dataset .\n MINUS { ?x schema:license ?this .}\n}\n" + } + ] + } + ], + "property": [ + { + "nodeKind": "sh:Literal", + "path": "schema:isBasedOn", + "datatype": { + "@id": "xsd:string" + }, + "maxCount": 1 + }, + { + "nodeKind": "sh:Literal", + "path": "schema:dateCreated", + "datatype": { + "@id": "xsd:string" + }, + "minCount": 1, + "maxCount": 1, + "sh:lessThanOrEquals": { + "@id": "schema:datePublished" + } + }, + { + "path": "schema:creator", + "sh:class": { + "@id": "schema:Person" + }, + "minCount": 1 + }, + { + "nodeKind": "sh:Literal", + "path": "schema:datePublished", + "datatype": { + "@id": "xsd:string" + }, + "maxCount": 1 + }, + { + "nodeKind": "sh:Literal", + "path": "schema:description", + "datatype": { + "@id": "xsd:string" + }, + "minCount": 1, + "maxCount": 1 + }, + { + "nodeKind": "sh:Literal", + "path": "schema:identifier", + "datatype": { + "@id": "xsd:string" + }, + "minCount": 1, + "maxCount": 1 + }, + { + "nodeKind": "sh:Literal", + "path": "schema:keywords", + "datatype": { + "@id": "xsd:string" + } + }, + { + "nodeKind": "sh:Literal", + "path": "schema:name", + "datatype": { + "@id": "xsd:string" + }, + "minCount": 1, + "maxCount": 1 + }, + { + "nodeKind": "sh:Literal", + "path": "prov:atLocation", + "datatype": { + "@id": "xsd:string" + }, + "minCount": 1, + "maxCount": 1 + }, + { + "nodeKind": "sh:Literal", + "path": "schema:sameAs", + "datatype": { + "@id": "xsd:string" + } + }, + { + "nodeKind": "sh:Literal", + "path": "schema:url", + "datatype": { + "@id": "xsd:string" + } + }, + { + "nodeKind": "sh:Literal", + "path": "schema:version", + "datatype": { + "@id": "xsd:string" + } + }, + { + "path": "schema:isPartOf", + "sh:class": { + "@id": "schema:Project" + }, + "minCount": 1, + "maxCount": 1 + }, + { + "path": "schema:subjectOf", + "sh:class": { + "@id": "schema:PublicationEvent" + } + }, + { + "path": "schema:hasPart", + "sh:class": { + "@id": "schema:DigitalDocument" + } + }, + { + "path": "schema:inLanguage", + "sh:class": { + "@id": "schema:Language" + } + }, + { + "nodeKind": "sh:Literal", + "path": "rdfs:label", + "datatype": { + "@id": "xsd:string" + } + }, + { + "path": "prov:qualifiedGeneration", + "sh:class": { + "@id": "prov:Generation" + } + } + ] + }, + { + "@id": "_:inLanguageShape", + "@type": "sh:NodeShape", + "ignoredProperties": [ + { + "@id": "rdf:type" + } + ], + "closed": true, + "targetClass": "schema:Language", + "property": [ + { + "nodeKind": "sh:Literal", + "path": "schema:name", + "datatype": { + "@id": "xsd:string" + }, + "minCount": 1, + "maxCount": 1 + }, + { + "nodeKind": "sh:Literal", + "path": "schema:alternateName", + "datatype": { + "@id": "xsd:string" + } + } + ] + }, + { + "@id": "_:datasetFileShape", + "@type": "sh:NodeShape", + "ignoredProperties": [ + { + "@id": "rdf:type" + } + ], + "closed": true, + "targetClass": "schema:DigitalDocument", + "property": [ + { + "nodeKind": "sh:Literal", + "path": "schema:name", + "datatype": { + "@id": "xsd:string" + }, + "minCount": 1, + "maxCount": 1 + }, + { + "nodeKind": "sh:Literal", + "path": "schema:dateCreated", + "datatype": { + "@id": "xsd:string" + }, + "minCount": 1, + "maxCount": 1 + }, + { + "nodeKind": "sh:Literal", + "path": "schema:url", + "datatype": { + "@id": "xsd:string" + }, + "minCount": 1, + "maxCount": 1 + }, + { + "nodeKind": "sh:Literal", + "path": "prov:atLocation", + "datatype": { + "@id": "xsd:string" + }, + "minCount": 1, + "maxCount": 1 + }, + { + "path": "schema:isPartOf", + "or": [ + { + "sh:class": { + "@id": "schema:Project" + } + }, + { + "nodeKind": "sh:Literal", + "datatype": { + "@id": "xsd:string" + } + } + ] + }, + { + "path": "schema:creator", + "sh:class": { + "@id": "schema:Person" + }, + "minCount": 1 + }, + { + "nodeKind": "sh:Literal", + "path": "rdfs:label", + "datatype": { + "@id": "xsd:string" + } + } + ] + }, + { + "@id": "_:datasetTagShape", + "@type": "sh:NodeShape", + "ignoredProperties": [ + { + "@id": "rdf:type" + } + ], + "closed": true, + "targetClass": "schema:PublicationEvent", + "property": [ + { + "nodeKind": "sh:Literal", + "path": "schema:name", + "datatype": { + "@id": "xsd:string" + }, + "minCount": 1, + "maxCount": 1 + }, + { + "nodeKind": "sh:Literal", + "path": "schema:description", + "datatype": { + "@id": "xsd:string" + }, + "minCount": 1, + "maxCount": 1 + }, + { + "nodeKind": "sh:Literal", + "path": "schema:startDate", + "datatype": { + "@id": "xsd:string" + }, + "minCount": 1, + "maxCount": 1 + }, + { + "nodeKind": "sh:Literal", + "path": "schema:location", + "datatype": { + "@id": "xsd:string" + }, + "minCount": 1, + "maxCount": 1 + }, + { + "nodeKind": "sh:Literal", + "path": "schema:about", + "datatype": { + "@id": "xsd:string" + }, + "minCount": 1, + "maxCount": 1 + } + ] + }, + { + "@id": "_:activityShape", + "@type": "sh:NodeShape", + "ignoredProperties": [ + { + "@id": "rdf:type" + } + ], + "closed": true, + "targetClass": "prov:Activity", + "property": [ + { + "path": "schema:isPartOf", + "sh:class": { + "@id": "schema:Project" + }, + "minCount": 1, + "maxCount": 1 + }, + { + "nodeKind": "sh:Literal", + "path": "rdfs:comment", + "datatype": { + "@id": "xsd:string" + }, + "minCount": 1, + "maxCount": 1 + }, + { + "nodeKind": "sh:Literal", + "path": "rdfs:label", + "datatype": { + "@id": "xsd:string" + } + }, + { + "nodeKind": "sh:IRI", + "path": "prov:wasInformedBy", + "minCount": 1, + "maxCount": 1 + }, + { + "nodeKind": "sh:Literal", + "path": "prov:influenced" + }, + { + "nodeKind": "sh:Literal", + "path": "prov:startedAtTime", + "datatype": { + "@id": "xsd:dateTime" + }, + "minCount": 1, + "maxCount": 1 + }, + { + "nodeKind": "sh:Literal", + "path": "prov:endedAtTime", + "datatype": { + "@id": "xsd:dateTime" + }, + "minCount": 1, + "maxCount": 1 + }, + { + "path": "prov:agent", + "or": [ + { + "sh:class": { + "@id": "prov:SoftwareAgent" + } + }, + { + "sh:class": { + "@id": "schema:Person" + } + }, + { + "nodeKind": "sh:IRI" + } + ], + "minCount": 2, + "maxCount": 2 + }, + { + "nodeKind": "sh:Literal", + "path": "prov:atLocation", + "datatype": { + "@id": "xsd:string" + } + }, + { + "path": "prov:qualifiedUsage", + "sh:class": { + "@id": "prov:Usage" + } + }, + { + "path": "prov:qualifiedAssociation", + "sh:class": { + "@id": "prov:Association" + } + }, + { + "path": "wfprov:wasPartOfWorkflowRun", + "sh:class": { + "@id": "wfprov:WorkflowRun" + } + } + ] + }, + { + "@id": "_:associationShape", + "@type": "sh:NodeShape", + "ignoredProperties": [ + { + "@id": "rdf:type" + } + ], + "closed": true, + "targetClass": "prov:Association", + "property": [ + { + "path": "prov:hadPlan", + "minCount": 1 + }, + { + "path": "prov:agent", + "sh:class": { + "@id": "prov:SoftwareAgent" + }, + "minCount": 1, + "maxCount": 1 + } + ] + }, + { + "@id": "_:usageShape", + "@type": "sh:NodeShape", + "ignoredProperties": [ + { + "@id": "rdf:type" + } + ], + "closed": true, + "targetClass": "prov:Usage", + "property": [ + { + "path": "prov:entity", + "minCount": 1 + }, + { + "nodeKind": "sh:Literal", + "path": "prov:hadRole", + "datatype": { + "@id": "xsd:string" + } + } + ] + }, + { + "@id": "_:softwareAgentShape", + "@type": "sh:NodeShape", + "ignoredProperties": [ + { + "@id": "rdf:type" + } + ], + "closed": true, + "targetClass": "prov:SoftwareAgent", + "property": [ + { + "nodeKind": "sh:Literal", + "path": "rdfs:label", + "datatype": { + "@id": "xsd:string" + } + }, + { + "path": "prov:wasStartedBy", + "or": [ + { + "nodeKind": "sh:IRI" + }, + { + "sh:class": { + "@id": "prov:Person" + } + }], + "maxCount": 1 + } + ] + }, + { + "@id": "_:generationShape", + "@type": "sh:NodeShape", + "ignoredProperties": [ + { + "@id": "rdf:type" + } + ], + "closed": true, + "targetClass": "prov:Generation", + "property": [ + { + "path": { + "sh:inversePath": { + "@id": "prov:qualifiedGeneration" + } + }, + "nodeKind": "sh:BlankNodeOrIRI" + }, + { + "nodeKind": "sh:Literal", + "path": "prov:hadRole", + "datatype": { + "@id": "xsd:string" + } + }, + { + "sh:class": { + "@id": "prov:Activity" + }, + "path": "prov:activity", + "minCount": 1 + } + ] + } + ] +} diff --git a/setup.py b/setup.py index 304b36d25a..ae0d86b42e 100644 --- a/setup.py +++ b/setup.py @@ -87,6 +87,7 @@ 'PyYAML>=3.12', 'pyld>=1.0.3', 'pyOpenSSL>=19.0.0', + 'pyshacl>=0.11.3.post1', 'python-dateutil>=2.6.1', 'python-editor>=1.0.4', 'rdflib-jsonld>=0.4.0', diff --git a/tests/cli/test_integration_datasets.py b/tests/cli/test_integration_datasets.py index d9b83e8eaa..5fe9e2ff05 100644 --- a/tests/cli/test_integration_datasets.py +++ b/tests/cli/test_integration_datasets.py @@ -22,7 +22,6 @@ import git import pytest -import yaml from renku.cli import cli @@ -596,13 +595,11 @@ def test_usage_error_in_add_from_git(runner, client, params, n_urls, message): def read_dataset_file_metadata(client, dataset_name, filename): """Return metadata from dataset's YAML file.""" - path = client.dataset_path(dataset_name) - assert path.exists() + with client.with_dataset(dataset_name) as dataset: + assert client.dataset_path(dataset.name).exists() - with path.open(mode='r') as fp: - metadata = yaml.safe_load(fp) - for file_ in metadata['files']: - if file_['path'].endswith(filename): + for file_ in dataset.files: + if file_.path.endswith(filename): return file_ @@ -631,14 +628,14 @@ def test_dataset_update(client, runner, params): assert 0 == result.exit_code after = read_dataset_file_metadata(client, 'remote', 'CHANGES.rst') - assert after['_id'] == before['_id'] - assert after['_label'] != before['_label'] - assert after['added'] == before['added'] - assert after['url'] == before['url'] - assert after['based_on']['_id'] == before['based_on']['_id'] - assert after['based_on']['_label'] != before['based_on']['_label'] - assert after['based_on']['path'] == before['based_on']['path'] - assert after['based_on']['based_on'] is None + assert after._id == before._id + assert after._label != before._label + assert after.added == before.added + assert after.url == before.url + assert after.based_on._id == before.based_on._id + assert after.based_on._label != before.based_on._label + assert after.based_on.path == before.based_on.path + assert after.based_on.based_on is None @pytest.mark.integration @@ -792,12 +789,12 @@ def test_import_from_renku_project(tmpdir, client, runner): assert 0 == result.exit_code metadata = read_dataset_file_metadata(client, 'remote-dataset', 'file') - assert metadata['creator'][0]['name'] == remote['creator'][0]['name'] - assert metadata['based_on']['_id'] == remote['_id'] - assert metadata['based_on']['_label'] == remote['_label'] - assert metadata['based_on']['path'] == remote['path'] - assert metadata['based_on']['based_on'] is None - assert metadata['based_on']['url'] == REMOTE + assert metadata.creator[0].name == remote.creator[0].name + assert metadata.based_on._id == remote._id + assert metadata.based_on._label == remote._label + assert metadata.based_on.path == remote.path + assert metadata.based_on.based_on is None + assert metadata.based_on.url == REMOTE @pytest.mark.integration diff --git a/tests/cli/test_log.py b/tests/cli/test_log.py new file mode 100644 index 0000000000..a561c4d353 --- /dev/null +++ b/tests/cli/test_log.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2017-2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Test ``log`` command.""" + +from __future__ import absolute_import, print_function + +import pytest + +from renku.cli import cli + + +@pytest.mark.shelled +@pytest.mark.parametrize('format', ['json-ld', 'nt', 'rdf']) +def test_run_log_strict(runner, project, run_shell, format): + """Test log output of run command.""" + # Run a shell command with pipe. + result = run_shell('renku run echo "a" > output') + + # Assert created output file. + result = runner.invoke( + cli, ['log', '--strict', '--format={}'.format(format)] + ) + assert 0 == result.exit_code, result.output + assert '.renku/workflow/' in result.output + + +@pytest.mark.shelled +@pytest.mark.parametrize('format', ['json-ld', 'nt', 'rdf']) +def test_dataset_log_strict(tmpdir, runner, project, client, format): + """Test output of log for dataset add.""" + result = runner.invoke(cli, ['dataset', 'create', 'my-dataset']) + assert 0 == result.exit_code + + paths = [] + test_paths = [] + for i in range(3): + new_file = tmpdir.join('file_{0}'.format(i)) + new_file.write(str(i)) + paths.append(str(new_file)) + test_paths.append(str(new_file.relto(tmpdir.join('..')))) + + # add data + result = runner.invoke( + cli, + ['dataset', 'add', 'my-dataset'] + paths, + ) + assert 0 == result.exit_code + + result = runner.invoke( + cli, ['log', '--strict', '--format={}'.format(format)] + ) + + assert 0 == result.exit_code, result.output + assert all(p in result.output for p in test_paths) diff --git a/tests/cli/test_update.py b/tests/cli/test_update.py index 59593dcc38..b0f5080b3c 100644 --- a/tests/cli/test_update.py +++ b/tests/cli/test_update.py @@ -35,6 +35,8 @@ def update_and_commit(data, file_, repo): def test_update(runner, project, run): """Test automatic file update.""" + from renku.core.utils.shacl import validate_graph + cwd = Path(project) data = cwd / 'data' data.mkdir() @@ -91,9 +93,13 @@ def test_update(runner, project, run): ['log', '--format', output_format], catch_exceptions=False, ) - assert 0 == result.exit_code, output_format + assert 0 == result.exit_code, result.output assert source.name in result.output, output_format + if output_format == 'nt': + r, _, t = validate_graph(result.output, format='nt') + assert r is True, t + def test_workflow_without_outputs(runner, project, run): """Test workflow without outputs.""" diff --git a/tests/core/models/test_shacl_schema.py b/tests/core/models/test_shacl_schema.py new file mode 100644 index 0000000000..fd6d6f1e01 --- /dev/null +++ b/tests/core/models/test_shacl_schema.py @@ -0,0 +1,106 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2017-2019- Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""test KG against SHACL shape.""" + +from renku.cli import cli +from renku.core.compat import Path, pyld +from renku.core.utils.shacl import validate_graph + + +def test_dataset_shacl(tmpdir, runner, project, client): + """Test dataset metadata structure.""" + force_dataset_path = Path( + __file__ + ).parent.parent.parent / 'fixtures' / 'force_dataset_shacl.json' + + force_datasetfile_path = Path( + __file__ + ).parent.parent.parent / 'fixtures' / 'force_datasetfile_shacl.json' + + force_datasettag_path = Path( + __file__ + ).parent.parent.parent / 'fixtures' / 'force_datasettag_shacl.json' + + runner.invoke(cli, ['dataset', 'create', 'dataset']) + + paths = [] + for i in range(3): + new_file = tmpdir.join('file_{0}'.format(i)) + new_file.write(str(i)) + paths.append(str(new_file)) + + # add data + runner.invoke( + cli, + ['dataset', 'add', 'dataset'] + paths, + catch_exceptions=False, + ) + + runner.invoke( + cli, + ['dataset', 'tag', 'dataset', '1.0'], + catch_exceptions=False, + ) + + with client.with_dataset('dataset') as dataset: + g = dataset.asjsonld() + rdf = pyld.jsonld.to_rdf( + g, + options={ + 'format': 'application/n-quads', + 'produceGeneralizedRdf': True + } + ) + + r, _, t = validate_graph(rdf, shacl_path=str(force_dataset_path)) + assert r is True, t + + r, _, t = validate_graph(rdf, shacl_path=str(force_datasetfile_path)) + assert r is True, t + + r, _, t = validate_graph(rdf, shacl_path=str(force_datasettag_path)) + assert r is True, t + + r, _, t = validate_graph(rdf) + assert r is True, t + + +def test_project_shacl(project, client): + """Test project metadata structure.""" + from renku.core.models.provenance.agents import Person + + path = Path( + __file__ + ).parent.parent.parent / 'fixtures' / 'force_project_shacl.json' + + project = client.project + project.creator = Person(email='johndoe@example.com', name='Johnny Doe') + + g = project.asjsonld() + rdf = pyld.jsonld.to_rdf( + g, + options={ + 'format': 'application/n-quads', + 'produceGeneralizedRdf': False + } + ) + r, _, t = validate_graph(rdf, shacl_path=str(path)) + assert r is True, t + + r, _, t = validate_graph(rdf) + assert r is True, t diff --git a/tests/fixtures/force_dataset_shacl.json b/tests/fixtures/force_dataset_shacl.json new file mode 100644 index 0000000000..3c201f0869 --- /dev/null +++ b/tests/fixtures/force_dataset_shacl.json @@ -0,0 +1,32 @@ +{ + "@context": { + "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + "sh": "http://www.w3.org/ns/shacl#", + "schema": "http://schema.org/" + }, + "@graph": [ + { + "@id": "_:forceDatasetShape", + "@type": "sh:NodeShape", + "sh:targetNode": { + "@id": "schema:Dataset", + "@type": "@id" + }, + "sh:property": [ + { + "sh:path": [ + { + "sh:inversePath": [ + { + "@id": "rdf:type", + "@type": "@id" + } + ] + } + ], + "sh:minCount": 1 + } + ] + } + ] +} diff --git a/tests/fixtures/force_datasetfile_shacl.json b/tests/fixtures/force_datasetfile_shacl.json new file mode 100644 index 0000000000..45470e3740 --- /dev/null +++ b/tests/fixtures/force_datasetfile_shacl.json @@ -0,0 +1,32 @@ +{ + "@context": { + "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + "sh": "http://www.w3.org/ns/shacl#", + "schema": "http://schema.org/" + }, + "@graph": [ + { + "@id": "_:forceDatasetShape", + "@type": "sh:NodeShape", + "sh:targetNode": { + "@id": "schema:DigitalDocument", + "@type": "@id" + }, + "sh:property": [ + { + "sh:path": [ + { + "sh:inversePath": [ + { + "@id": "rdf:type", + "@type": "@id" + } + ] + } + ], + "sh:minCount": 1 + } + ] + } + ] +} diff --git a/tests/fixtures/force_datasettag_shacl.json b/tests/fixtures/force_datasettag_shacl.json new file mode 100644 index 0000000000..106f5e0e41 --- /dev/null +++ b/tests/fixtures/force_datasettag_shacl.json @@ -0,0 +1,32 @@ +{ + "@context": { + "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + "sh": "http://www.w3.org/ns/shacl#", + "schema": "http://schema.org/" + }, + "@graph": [ + { + "@id": "_:forceDatasetShape", + "@type": "sh:NodeShape", + "sh:targetNode": { + "@id": "schema:PublicationEvent", + "@type": "@id" + }, + "sh:property": [ + { + "sh:path": [ + { + "sh:inversePath": [ + { + "@id": "rdf:type", + "@type": "@id" + } + ] + } + ], + "sh:minCount": 1 + } + ] + } + ] +} diff --git a/tests/fixtures/force_project_shacl.json b/tests/fixtures/force_project_shacl.json new file mode 100644 index 0000000000..b7fd526983 --- /dev/null +++ b/tests/fixtures/force_project_shacl.json @@ -0,0 +1,32 @@ +{ + "@context": { + "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + "sh": "http://www.w3.org/ns/shacl#", + "schema": "http://schema.org/" + }, + "@graph": [ + { + "@id": "_:forceProjectShape", + "@type": "sh:NodeShape", + "sh:targetNode": { + "@id": "schema:Project", + "@type": "@id" + }, + "sh:property": [ + { + "sh:path": [ + { + "sh:inversePath": [ + { + "@id": "rdf:type", + "@type": "@id" + } + ] + } + ], + "sh:minCount": 1 + } + ] + } + ] +} From f11edf29bf436d5db32ea61b0fb92887056c0fc7 Mon Sep 17 00:00:00 2001 From: Ralf Grubenmann Date: Fri, 29 Nov 2019 16:47:30 +0100 Subject: [PATCH 02/11] Fix JSON-LD translation and related issues (#846) * Add default value converter and remove dataset property from DatasetFile * Removes type from Person context, fixes missing context in jsonld translation of project, fixes url encoding of dataset names --- renku/core/commands/checks/migration.py | 3 ++- renku/core/commands/providers/dataverse.py | 1 - renku/core/commands/providers/zenodo.py | 1 - renku/core/management/datasets.py | 5 +---- renku/core/models/datasets.py | 2 -- renku/core/models/jsonld.py | 21 +++++++++++++++++++++ renku/core/models/migrations/dataset.py | 11 ++++++++--- renku/core/models/projects.py | 3 ++- 8 files changed, 34 insertions(+), 13 deletions(-) diff --git a/renku/core/commands/checks/migration.py b/renku/core/commands/checks/migration.py index f1583b3f06..6bec34afce 100644 --- a/renku/core/commands/checks/migration.py +++ b/renku/core/commands/checks/migration.py @@ -240,7 +240,8 @@ def fix_dataset_files_urls(client): """Ensure dataset files have correct url format.""" for dataset in client.datasets.values(): for file_ in dataset.files: - file_.url = url_to_string(file_.url) + if file_.url: + file_.url = url_to_string(file_.url) dataset.to_yaml() diff --git a/renku/core/commands/providers/dataverse.py b/renku/core/commands/providers/dataverse.py index 660c6e5785..7f912c47e9 100644 --- a/renku/core/commands/providers/dataverse.py +++ b/renku/core/commands/providers/dataverse.py @@ -220,7 +220,6 @@ def as_dataset(self, client): filename=file_.name, filesize=file_.content_size, filetype=file_.file_format, - dataset=dataset.name, path='', ) serialized_files.append(dataset_file) diff --git a/renku/core/commands/providers/zenodo.py b/renku/core/commands/providers/zenodo.py index 66bb45927c..51b328b232 100644 --- a/renku/core/commands/providers/zenodo.py +++ b/renku/core/commands/providers/zenodo.py @@ -245,7 +245,6 @@ def as_dataset(self, client): filename=file_.filename, filesize=file_.filesize, filetype=file_.type, - dataset=dataset.name, path='', ) serialized_files.append(dataset_file) diff --git a/renku/core/management/datasets.py b/renku/core/management/datasets.py index 426bc70a1e..b48af3e039 100644 --- a/renku/core/management/datasets.py +++ b/renku/core/management/datasets.py @@ -304,7 +304,6 @@ def _add_from_local(self, dataset, path, link, destination): 'path': path_in_repo, 'url': path_in_repo, 'creator': dataset.creator, - 'dataset': dataset.name, 'parent': self }] @@ -326,7 +325,6 @@ def _add_from_local(self, dataset, path, link, destination): 'path': destination.relative_to(self.path), 'url': 'file://' + os.path.relpath(str(src), str(self.path)), 'creator': dataset.creator, - 'dataset': dataset.name, 'parent': self }] @@ -352,7 +350,6 @@ def _add_from_url(self, dataset, url, destination): 'path': destination.relative_to(self.path), 'url': remove_credentials(url), 'creator': dataset.creator, - 'dataset': dataset.name, 'parent': self }] @@ -361,6 +358,7 @@ def _add_from_git(self, dataset, url, sources, destination, ref): from renku import LocalClient u = parse.urlparse(url) + sources = self._resolve_paths(u.path, sources) # Get all files from repo that match sources @@ -429,7 +427,6 @@ def _add_from_git(self, dataset, url, sources, destination, ref): 'path': path_in_dst_repo, 'url': remove_credentials(url), 'creator': creators, - 'dataset': dataset.name, 'parent': self, 'based_on': based_on }) diff --git a/renku/core/models/datasets.py b/renku/core/models/datasets.py index 22295b5e6c..8ac9989ba2 100644 --- a/renku/core/models/datasets.py +++ b/renku/core/models/datasets.py @@ -172,8 +172,6 @@ class DatasetFile(Entity, CreatorMixin): checksum = attr.ib(default=None, kw_only=True) - dataset = jsonld.ib(context='schema:isPartOf', default=None, kw_only=True) - filename = attr.ib(kw_only=True, converter=convert_filename_path) name = jsonld.ib(context='schema:name', kw_only=True, default=None) diff --git a/renku/core/models/jsonld.py b/renku/core/models/jsonld.py index 5e2a9adbd8..3f47488218 100644 --- a/renku/core/models/jsonld.py +++ b/renku/core/models/jsonld.py @@ -22,6 +22,7 @@ import weakref from copy import deepcopy from datetime import datetime, timezone +from functools import partial from importlib import import_module from pathlib import Path @@ -265,12 +266,23 @@ def _propagate_reference_contexts( return current_context, scoped_properties +def _default_converter(cls, value): + """A default converter method that tries to deserialize objects.""" + if isinstance(value, dict): + return cls.from_jsonld(value) + + return value + + def attrib(context=None, type=None, **kwargs): """Create a new attribute with context.""" kwargs.setdefault('metadata', {}) kwargs['metadata'][KEY] = context if type: kwargs['metadata'][KEY_CLS] = type + + if 'converter' not in kwargs and hasattr(type, 'from_jsonld'): + kwargs['converter'] = partial(_default_converter, type) return attr.ib(**kwargs) @@ -523,6 +535,15 @@ def from_jsonld( for k, v in compacted.items(): if k in fields: + no_value_context = isinstance(v, dict) and '@context' not in v + has_nested_context = ( + k in compacted['@context'] and + '@context' in compacted['@context'][k] + ) + if no_value_context and has_nested_context: + # Propagate down context + v['@context'] = compacted['@context'][k]['@context'] + data_[k.lstrip('_')] = v if __reference__: diff --git a/renku/core/models/migrations/dataset.py b/renku/core/models/migrations/dataset.py index ca5c47132f..e9e7b61084 100644 --- a/renku/core/models/migrations/dataset.py +++ b/renku/core/models/migrations/dataset.py @@ -30,7 +30,12 @@ def migrate_dataset_schema(data): ) data['creator'] = data.pop('authors', {}) - for file_name, file_ in data.get('files', {}).items(): + + files = data.get('files', []) + + if isinstance(files, dict): + files = files.values() + for file_ in files: file_['creator'] = file_.pop('authors', {}) return data @@ -52,13 +57,13 @@ def migrate_absolute_paths(data): files = data.get('files', []) if isinstance(files, dict): - files = files.values() + files = list(files.values()) for file_ in files: path = Path(file_.get('path'), '.') if path.is_absolute(): file_['path'] = path.relative_to((os.getcwd())) - + data['files'] = files return data diff --git a/renku/core/models/projects.py b/renku/core/models/projects.py index c53eb6ce16..1ef917609f 100644 --- a/renku/core/models/projects.py +++ b/renku/core/models/projects.py @@ -73,7 +73,6 @@ class Project(object): kw_only=True, context={ '@id': 'schema:creator', - '@type': 'schema:Person', }, type=Person ) @@ -120,6 +119,8 @@ def project_id(self): owner = remote.get('owner') or owner name = remote.get('name') or name host = os.environ.get('RENKU_DOMAIN') or host + if name: + name = urllib.parse.quote(name, safe='') project_url = urllib.parse.urljoin( 'https://{host}'.format(host=host), pathlib.posixpath.join(PROJECT_URL_PATH, owner, name or 'NULL') From 4a1ac4af285d8c172818093bc645f826541434de Mon Sep 17 00:00:00 2001 From: sam Date: Wed, 23 Oct 2019 21:24:43 +0200 Subject: [PATCH 03/11] feat: added renku service with cache and datasets --- .env | 4 + .gitignore | 2 + .travis.yml | 2 +- Dockerfile => Dockerfile.cli | 2 +- Dockerfile.svc | 18 + MANIFEST.in | 6 +- Makefile | 3 + Pipfile.lock | 431 ++++++++++-------- conftest.py | 175 ++++++++ docker-compose.yml | 13 + renku/cli/__init__.py | 2 +- renku/core/commands/client.py | 22 +- renku/core/commands/clone.py | 12 +- renku/core/commands/dataset.py | 2 +- renku/core/management/clone.py | 32 +- renku/core/management/repository.py | 7 +- renku/core/utils/contexts.py | 3 + renku/service/.env-example | 7 + renku/service/__init__.py | 18 + renku/service/cache/__init__.py | 26 ++ renku/service/cache/base.py | 63 +++ renku/service/cache/config.py | 24 ++ renku/service/cache/files.py | 51 +++ renku/service/cache/projects.py | 46 ++ renku/service/config.py | 57 +++ renku/service/entrypoint.py | 102 +++++ renku/service/serializers/__init__.py | 18 + renku/service/serializers/cache.py | 171 ++++++++ renku/service/serializers/datasets.py | 131 ++++++ renku/service/serializers/headers.py | 58 +++ renku/service/serializers/rpc.py | 25 ++ renku/service/utils/__init__.py | 55 +++ renku/service/views/__init__.py | 18 + renku/service/views/cache.py | 244 +++++++++++ renku/service/views/datasets.py | 235 ++++++++++ renku/service/views/decorators.py | 250 +++++++++++ setup.py | 8 + tests/cli/test_datasets.py | 4 +- tests/service/test_cache_views.py | 599 ++++++++++++++++++++++++++ tests/service/test_dataset_views.py | 547 +++++++++++++++++++++++ tests/service/test_exceptions.py | 62 +++ 41 files changed, 3366 insertions(+), 189 deletions(-) create mode 100644 .env rename Dockerfile => Dockerfile.cli (96%) create mode 100644 Dockerfile.svc create mode 100644 docker-compose.yml create mode 100644 renku/service/.env-example create mode 100644 renku/service/__init__.py create mode 100644 renku/service/cache/__init__.py create mode 100644 renku/service/cache/base.py create mode 100644 renku/service/cache/config.py create mode 100644 renku/service/cache/files.py create mode 100644 renku/service/cache/projects.py create mode 100644 renku/service/config.py create mode 100644 renku/service/entrypoint.py create mode 100644 renku/service/serializers/__init__.py create mode 100644 renku/service/serializers/cache.py create mode 100644 renku/service/serializers/datasets.py create mode 100644 renku/service/serializers/headers.py create mode 100644 renku/service/serializers/rpc.py create mode 100644 renku/service/utils/__init__.py create mode 100644 renku/service/views/__init__.py create mode 100644 renku/service/views/cache.py create mode 100644 renku/service/views/datasets.py create mode 100644 renku/service/views/decorators.py create mode 100644 tests/service/test_cache_views.py create mode 100644 tests/service/test_dataset_views.py create mode 100644 tests/service/test_exceptions.py diff --git a/.env b/.env new file mode 100644 index 0000000000..222989c353 --- /dev/null +++ b/.env @@ -0,0 +1,4 @@ +REDIS_HOST=redis +REDIS_PORT=6379 +REDIS_DATABASE=0 +REDIS_PASSWORD= diff --git a/.gitignore b/.gitignore index 9c0ccc6751..56245fde97 100644 --- a/.gitignore +++ b/.gitignore @@ -73,3 +73,5 @@ target/ renku-*.bottle.json renku-*.bottle.tar.gz renku.rb + +.env diff --git a/.travis.yml b/.travis.yml index db5a1653d8..70eaaa17be 100644 --- a/.travis.yml +++ b/.travis.yml @@ -60,7 +60,7 @@ before_install: - if [[ $TRAVIS_OS_NAME == 'linux' ]]; then sudo apt-get update; sudo apt-get -y install shellcheck; - travis_retry python -m pip install --upgrade pip setuptools py; + travis_retry python -m pip install --upgrade six pip setuptools py; travis_retry python -m pip install twine wheel coveralls requirements-builder; requirements-builder -e all --level=min setup.py > .travis-lowest-requirements.txt; requirements-builder -e all --level=pypi setup.py > .travis-release-requirements.txt; diff --git a/Dockerfile b/Dockerfile.cli similarity index 96% rename from Dockerfile rename to Dockerfile.cli index 770a6e39ee..6e63b77185 100644 --- a/Dockerfile +++ b/Dockerfile.cli @@ -1,4 +1,4 @@ -FROM python:3.6-alpine as base +FROM python:3.7-alpine as base RUN apk add --no-cache git && \ pip install --no-cache --upgrade pip diff --git a/Dockerfile.svc b/Dockerfile.svc new file mode 100644 index 0000000000..dc7283df4d --- /dev/null +++ b/Dockerfile.svc @@ -0,0 +1,18 @@ +FROM python:3.7-alpine + +RUN apk add --update --no-cache alpine-sdk g++ gcc linux-headers libxslt-dev python3-dev build-base openssl-dev libffi-dev git bash && \ + pip install --no-cache --upgrade pip setuptools pipenv requirements-builder + +RUN apk add --no-cache --allow-untrusted \ + --repository http://dl-cdn.alpinelinux.org/alpine/latest-stable/community \ + --repository http://dl-cdn.alpinelinux.org/alpine/latest-stable/main \ + --repository http://nl.alpinelinux.org/alpine/edge/community \ + git-lfs && \ + git lfs install + +COPY . /code/renku +WORKDIR /code/renku +RUN requirements-builder -e all --level=pypi setup.py > requirements.txt && pip install -r requirements.txt && pip install -e . && pip install gunicorn + + +ENTRYPOINT ["gunicorn", "renku.service.entrypoint:app", "-b", "0.0.0.0:8080"] diff --git a/MANIFEST.in b/MANIFEST.in index 8c9e058fa7..d77b546611 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -17,15 +17,17 @@ # limitations under the License. # Check manifest will not automatically add these two files: +include renku/service/.env-example include .dockerignore include .editorconfig include .tx/config include *.md prune docs/_build recursive-include renku *.po *.pot *.mo - +recursive-include renku *.py # added by check_manifest.py include *.py +include *.yml include *.rst include *.sh include *.txt @@ -39,6 +41,7 @@ include babel.ini include brew.py include pytest.ini include snap/snapcraft.yaml +recursive-include renku *.json recursive-include .github CODEOWNERS recursive-include .travis *.sh recursive-include docs *.bat @@ -60,3 +63,4 @@ recursive-include renku *.json recursive-include renku Dockerfile recursive-include tests *.py *.gz *.yml *.json prune .github +prune .env diff --git a/Makefile b/Makefile index 2602ce90e6..9e04a816c8 100644 --- a/Makefile +++ b/Makefile @@ -69,3 +69,6 @@ brew-commit-bottle: *.bottle.json brew-release: open "https://github.com/SwissDataScienceCenter/renku-python/releases/new?tag=v$(shell brew info --json=v1 renku | jq -r '.[0].versions.stable')" + +service-container: + docker build -f Dockerfile.svc -t renku-svc:`git rev-parse --short HEAD` . diff --git a/Pipfile.lock b/Pipfile.lock index 936bc038a6..559749827d 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -30,6 +30,13 @@ ], "version": "==1.5" }, + "apispec": { + "hashes": [ + "sha256:5fdaa1173b32515cc83f9d413a49a6c37fafc2b87f6b40e95923d3e85f0942c5", + "sha256:9e88c51517a6515612e818459f61c1bc06c00f2313e5187828bdbabaa7461473" + ], + "version": "==3.0.0" + }, "appdirs": { "hashes": [ "sha256:9e5896d1372858f8dd3344faf4e5014d21849c756c8d5701f78f8a103b372d92", @@ -37,26 +44,12 @@ ], "version": "==1.4.3" }, - "asn1crypto": { - "hashes": [ - "sha256:0b199f211ae690df3db4fd6c1c4ff976497fb1da689193e368eedbadc53d9292", - "sha256:bca90060bd995c3f62c4433168eab407e44bdbdb567b3f3a396a676c1a4c4a3f" - ], - "version": "==1.0.1" - }, - "atomicwrites": { - "hashes": [ - "sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4", - "sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6" - ], - "version": "==1.3.0" - }, "attrs": { "hashes": [ - "sha256:ec20e7a4825331c1b5ebf261d111e16fa9612c1f7a5e1f884f12bd53a664dfd2", - "sha256:f913492e1663d3c36f502e5e9ba6cd13cf19d7fab50aa13239e420fef95e1396" + "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c", + "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72" ], - "version": "==19.2.0" + "version": "==19.3.0" }, "avro-cwl": { "hashes": [ @@ -87,43 +80,48 @@ }, "certifi": { "hashes": [ - "sha256:e4f3620cfea4f83eedc95b24abd9cd56f3c4b146dd0177e83a21b4eb49e21e50", - "sha256:fd7c7c74727ddcf00e9acd26bba8da604ffec95bf1c2144e67aff7a8b50e6cef" + "sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3", + "sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f" ], - "version": "==2019.9.11" + "version": "==2019.11.28" }, "cffi": { "hashes": [ - "sha256:041c81822e9f84b1d9c401182e174996f0bae9991f33725d059b771744290774", - "sha256:046ef9a22f5d3eed06334d01b1e836977eeef500d9b78e9ef693f9380ad0b83d", - "sha256:066bc4c7895c91812eff46f4b1c285220947d4aa46fa0a2651ff85f2afae9c90", - "sha256:066c7ff148ae33040c01058662d6752fd73fbc8e64787229ea8498c7d7f4041b", - "sha256:2444d0c61f03dcd26dbf7600cf64354376ee579acad77aef459e34efcb438c63", - "sha256:300832850b8f7967e278870c5d51e3819b9aad8f0a2c8dbe39ab11f119237f45", - "sha256:34c77afe85b6b9e967bd8154e3855e847b70ca42043db6ad17f26899a3df1b25", - "sha256:46de5fa00f7ac09f020729148ff632819649b3e05a007d286242c4882f7b1dc3", - "sha256:4aa8ee7ba27c472d429b980c51e714a24f47ca296d53f4d7868075b175866f4b", - "sha256:4d0004eb4351e35ed950c14c11e734182591465a33e960a4ab5e8d4f04d72647", - "sha256:4e3d3f31a1e202b0f5a35ba3bc4eb41e2fc2b11c1eff38b362de710bcffb5016", - "sha256:50bec6d35e6b1aaeb17f7c4e2b9374ebf95a8975d57863546fa83e8d31bdb8c4", - "sha256:55cad9a6df1e2a1d62063f79d0881a414a906a6962bc160ac968cc03ed3efcfb", - "sha256:5662ad4e4e84f1eaa8efce5da695c5d2e229c563f9d5ce5b0113f71321bcf753", - "sha256:59b4dc008f98fc6ee2bb4fd7fc786a8d70000d058c2bbe2698275bc53a8d3fa7", - "sha256:73e1ffefe05e4ccd7bcea61af76f36077b914f92b76f95ccf00b0c1b9186f3f9", - "sha256:a1f0fd46eba2d71ce1589f7e50a9e2ffaeb739fb2c11e8192aa2b45d5f6cc41f", - "sha256:a2e85dc204556657661051ff4bab75a84e968669765c8a2cd425918699c3d0e8", - "sha256:a5457d47dfff24882a21492e5815f891c0ca35fefae8aa742c6c263dac16ef1f", - "sha256:a8dccd61d52a8dae4a825cdbb7735da530179fea472903eb871a5513b5abbfdc", - "sha256:ae61af521ed676cf16ae94f30fe202781a38d7178b6b4ab622e4eec8cefaff42", - "sha256:b012a5edb48288f77a63dba0840c92d0504aa215612da4541b7b42d849bc83a3", - "sha256:d2c5cfa536227f57f97c92ac30c8109688ace8fa4ac086d19d0af47d134e2909", - "sha256:d42b5796e20aacc9d15e66befb7a345454eef794fdb0737d1af593447c6c8f45", - "sha256:dee54f5d30d775f525894d67b1495625dd9322945e7fee00731952e0368ff42d", - "sha256:e070535507bd6aa07124258171be2ee8dfc19119c28ca94c9dfb7efd23564512", - "sha256:e1ff2748c84d97b065cc95429814cdba39bcbd77c9c85c89344b317dc0d9cbff", - "sha256:ed851c75d1e0e043cbf5ca9a8e1b13c4c90f3fbd863dacb01c0808e2b5204201" - ], - "version": "==1.12.3" + "sha256:0b49274afc941c626b605fb59b59c3485c17dc776dc3cc7cc14aca74cc19cc42", + "sha256:0e3ea92942cb1168e38c05c1d56b0527ce31f1a370f6117f1d490b8dcd6b3a04", + "sha256:135f69aecbf4517d5b3d6429207b2dff49c876be724ac0c8bf8e1ea99df3d7e5", + "sha256:19db0cdd6e516f13329cba4903368bff9bb5a9331d3410b1b448daaadc495e54", + "sha256:2781e9ad0e9d47173c0093321bb5435a9dfae0ed6a762aabafa13108f5f7b2ba", + "sha256:291f7c42e21d72144bb1c1b2e825ec60f46d0a7468f5346841860454c7aa8f57", + "sha256:2c5e309ec482556397cb21ede0350c5e82f0eb2621de04b2633588d118da4396", + "sha256:2e9c80a8c3344a92cb04661115898a9129c074f7ab82011ef4b612f645939f12", + "sha256:32a262e2b90ffcfdd97c7a5e24a6012a43c61f1f5a57789ad80af1d26c6acd97", + "sha256:3c9fff570f13480b201e9ab69453108f6d98244a7f495e91b6c654a47486ba43", + "sha256:415bdc7ca8c1c634a6d7163d43fb0ea885a07e9618a64bda407e04b04333b7db", + "sha256:42194f54c11abc8583417a7cf4eaff544ce0de8187abaf5d29029c91b1725ad3", + "sha256:4424e42199e86b21fc4db83bd76909a6fc2a2aefb352cb5414833c030f6ed71b", + "sha256:4a43c91840bda5f55249413037b7a9b79c90b1184ed504883b72c4df70778579", + "sha256:599a1e8ff057ac530c9ad1778293c665cb81a791421f46922d80a86473c13346", + "sha256:5c4fae4e9cdd18c82ba3a134be256e98dc0596af1e7285a3d2602c97dcfa5159", + "sha256:5ecfa867dea6fabe2a58f03ac9186ea64da1386af2159196da51c4904e11d652", + "sha256:62f2578358d3a92e4ab2d830cd1c2049c9c0d0e6d3c58322993cc341bdeac22e", + "sha256:6471a82d5abea994e38d2c2abc77164b4f7fbaaf80261cb98394d5793f11b12a", + "sha256:6d4f18483d040e18546108eb13b1dfa1000a089bcf8529e30346116ea6240506", + "sha256:71a608532ab3bd26223c8d841dde43f3516aa5d2bf37b50ac410bb5e99053e8f", + "sha256:74a1d8c85fb6ff0b30fbfa8ad0ac23cd601a138f7509dc617ebc65ef305bb98d", + "sha256:7b93a885bb13073afb0aa73ad82059a4c41f4b7d8eb8368980448b52d4c7dc2c", + "sha256:7d4751da932caaec419d514eaa4215eaf14b612cff66398dd51129ac22680b20", + "sha256:7f627141a26b551bdebbc4855c1157feeef18241b4b8366ed22a5c7d672ef858", + "sha256:8169cf44dd8f9071b2b9248c35fc35e8677451c52f795daa2bb4643f32a540bc", + "sha256:aa00d66c0fab27373ae44ae26a66a9e43ff2a678bf63a9c7c1a9a4d61172827a", + "sha256:ccb032fda0873254380aa2bfad2582aedc2959186cce61e3a17abc1a55ff89c3", + "sha256:d754f39e0d1603b5b24a7f8484b22d2904fa551fe865fd0d4c3332f078d20d4e", + "sha256:d75c461e20e29afc0aee7172a0950157c704ff0dd51613506bd7d82b718e7410", + "sha256:dcd65317dd15bc0451f3e01c80da2216a31916bdcffd6221ca1202d96584aa25", + "sha256:e570d3ab32e2c2861c4ebe6ffcad6a8abf9347432a37608fe1fbd157b3f0036b", + "sha256:fd43a88e045cf992ed09fa724b5315b790525f2676883a6ea64e3263bae6549d" + ], + "version": "==1.13.2" }, "chardet": { "hashes": [ @@ -134,10 +132,10 @@ }, "check-manifest": { "hashes": [ - "sha256:8754cc8efd7c062a3705b442d1c23ff702d4477b41a269c2e354b25e1f5535a4", - "sha256:a4c555f658a7c135b8a22bd26c2e55cfaf5876e4d5962d8c25652f2addd556bc" + "sha256:42de6eaab4ed149e60c9b367ada54f01a3b1e4d6846784f9b9710e770ff5572c", + "sha256:78dd077f2c70dbac7cfcc9d12cbd423914e787ea4b5631de45aecd25b524e8e3" ], - "version": "==0.39" + "version": "==0.40" }, "click": { "hashes": [ @@ -148,9 +146,9 @@ }, "click-completion": { "hashes": [ - "sha256:78072eecd5e25ea0d25ceaf99cd5f22aa2667d67231ae0819deab9b1ff3456fb" + "sha256:5bf816b81367e638a190b6e91b50779007d14301b3f9f3145d68e3cade7bce86" ], - "version": "==0.5.1" + "version": "==0.5.2" }, "coverage": { "hashes": [ @@ -191,24 +189,29 @@ }, "cryptography": { "hashes": [ - "sha256:24b61e5fcb506424d3ec4e18bca995833839bf13c59fc43e530e488f28d46b8c", - "sha256:25dd1581a183e9e7a806fe0543f485103232f940fcfc301db65e630512cce643", - "sha256:3452bba7c21c69f2df772762be0066c7ed5dc65df494a1d53a58b683a83e1216", - "sha256:41a0be220dd1ed9e998f5891948306eb8c812b512dc398e5a01846d855050799", - "sha256:5751d8a11b956fbfa314f6553d186b94aa70fdb03d8a4d4f1c82dcacf0cbe28a", - "sha256:5f61c7d749048fa6e3322258b4263463bfccefecb0dd731b6561cb617a1d9bb9", - "sha256:72e24c521fa2106f19623a3851e9f89ddfdeb9ac63871c7643790f872a305dfc", - "sha256:7b97ae6ef5cba2e3bb14256625423413d5ce8d1abb91d4f29b6d1a081da765f8", - "sha256:961e886d8a3590fd2c723cf07be14e2a91cf53c25f02435c04d39e90780e3b53", - "sha256:96d8473848e984184b6728e2c9d391482008646276c3ff084a1bd89e15ff53a1", - "sha256:ae536da50c7ad1e002c3eee101871d93abdc90d9c5f651818450a0d3af718609", - "sha256:b0db0cecf396033abb4a93c95d1602f268b3a68bb0a9cc06a7cff587bb9a7292", - "sha256:cfee9164954c186b191b91d4193989ca994703b2fff406f71cf454a2d3c7327e", - "sha256:e6347742ac8f35ded4a46ff835c60e68c22a536a8ae5c4422966d06946b6d4c6", - "sha256:f27d93f0139a3c056172ebb5d4f9056e770fdf0206c2f422ff2ebbad142e09ed", - "sha256:f57b76e46a58b63d1c6375017f4564a28f19a5ca912691fd2e4261b3414b618d" - ], - "version": "==2.7" + "sha256:02079a6addc7b5140ba0825f542c0869ff4df9a69c360e339ecead5baefa843c", + "sha256:1df22371fbf2004c6f64e927668734070a8953362cd8370ddd336774d6743595", + "sha256:369d2346db5934345787451504853ad9d342d7f721ae82d098083e1f49a582ad", + "sha256:3cda1f0ed8747339bbdf71b9f38ca74c7b592f24f65cdb3ab3765e4b02871651", + "sha256:44ff04138935882fef7c686878e1c8fd80a723161ad6a98da31e14b7553170c2", + "sha256:4b1030728872c59687badcca1e225a9103440e467c17d6d1730ab3d2d64bfeff", + "sha256:58363dbd966afb4f89b3b11dfb8ff200058fbc3b947507675c19ceb46104b48d", + "sha256:6ec280fb24d27e3d97aa731e16207d58bd8ae94ef6eab97249a2afe4ba643d42", + "sha256:7270a6c29199adc1297776937a05b59720e8a782531f1f122f2eb8467f9aab4d", + "sha256:73fd30c57fa2d0a1d7a49c561c40c2f79c7d6c374cc7750e9ac7c99176f6428e", + "sha256:7f09806ed4fbea8f51585231ba742b58cbcfbfe823ea197d8c89a5e433c7e912", + "sha256:90df0cc93e1f8d2fba8365fb59a858f51a11a394d64dbf3ef844f783844cc793", + "sha256:971221ed40f058f5662a604bd1ae6e4521d84e6cad0b7b170564cc34169c8f13", + "sha256:a518c153a2b5ed6b8cc03f7ae79d5ffad7315ad4569b2d5333a13c38d64bd8d7", + "sha256:b0de590a8b0979649ebeef8bb9f54394d3a41f66c5584fff4220901739b6b2f0", + "sha256:b43f53f29816ba1db8525f006fa6f49292e9b029554b3eb56a189a70f2a40879", + "sha256:d31402aad60ed889c7e57934a03477b572a03af7794fa8fb1780f21ea8f6551f", + "sha256:de96157ec73458a7f14e3d26f17f8128c959084931e8997b9e655a39c8fde9f9", + "sha256:df6b4dca2e11865e6cfbfb708e800efb18370f5a46fd601d3755bc7f85b3a8a2", + "sha256:ecadccc7ba52193963c0475ac9f6fa28ac01e01349a2ca48509667ef41ffd2cf", + "sha256:fb81c17e0ebe3358486cd8cc3ad78adbae58af12fc2bf2bc0bb84e8090fa5ce8" + ], + "version": "==2.8" }, "cwlref-runner": { "hashes": [ @@ -226,10 +229,10 @@ }, "decorator": { "hashes": [ - "sha256:86156361c50488b84a3f148056ea716ca587df2f0de1d34750d35c21312725de", - "sha256:f069f3a01830ca754ba5258fde2278454a0b5b79e0d7f5c13b3b97e57d4acff6" + "sha256:54c38050039232e1db4ad7375cfce6748d7b41c29e95a081c8a6d2c30364a2ce", + "sha256:5d19b92a3c8f7f101c8dd86afd86b0f061a8ce4540ab8cd401fa2542756bce6d" ], - "version": "==4.4.0" + "version": "==4.4.1" }, "docutils": { "hashes": [ @@ -260,6 +263,13 @@ ], "version": "==1.7.1" }, + "fakeredis": { + "hashes": [ + "sha256:169598943dc10aadd62871a34b2867bb5e24f9da7ebc97a2058c3f35c760241e", + "sha256:1db27ec3a5c964b9fb9f36ec1b9770a81204c54e84f83c763f36689eef4a5fd4" + ], + "version": "==1.1.0" + }, "filelock": { "hashes": [ "sha256:18d82244ee114f543149c66a6e0c14e9c4f8a1044b5cdaadd0f82159d6a6ff59", @@ -269,10 +279,30 @@ }, "flake8": { "hashes": [ - "sha256:19241c1cbc971b9962473e4438a2ca19749a7dd002dd1a946eaba171b4114548", - "sha256:8e9dfa3cecb2400b3738a42c54c3043e821682b9c840b0448c0503f781130696" + "sha256:45681a117ecc81e870cbf1262835ae4af5e7a8b08e40b944a8a6e6b895914cfb", + "sha256:49356e766643ad15072a789a20915d3c91dc89fd313ccd71802303fd67e4deca" + ], + "version": "==3.7.9" + }, + "flask": { + "hashes": [ + "sha256:13f9f196f330c7c2c5d7a5cf91af894110ca0215ac051b5844701f2bfd934d52", + "sha256:45eb5a6fd193d6cf7e0cf5d8a5b31f83d5faae0293695626f539a823e93b13f6" + ], + "version": "==1.1.1" + }, + "flask-apispec": { + "hashes": [ + "sha256:46bb89f8c4be3547d3f48536100f88a2a249ae59b050589cff57a0ec8e25d000", + "sha256:b97a9d7200293021ff11fa393157f51736dc12d6b4fc4502140561fb3cf64a16" ], - "version": "==3.7.8" + "version": "==0.8.3" + }, + "flask-swagger-ui": { + "hashes": [ + "sha256:3282c770764c8053360f33b2fc120e1d169ecca2138537d0e6e1135b1f9d4ff2" + ], + "version": "==3.20.9" }, "freezegun": { "hashes": [ @@ -319,11 +349,11 @@ }, "importlib-metadata": { "hashes": [ - "sha256:aa18d7378b00b40847790e7c27e11673d7fed219354109d0e7b9e5b25dc3ad26", - "sha256:d5f18a79777f3aa179c145737780282e27b508fc8fd688cb17c7a813e8bd39af" + "sha256:b044f07694ef14a6683b097ba56bd081dbc7cdc7c7fe46011e499dfecc082f21", + "sha256:e6ac600a142cf2db707b1998382cc7fc3b02befb7273876e01b8ad10b9652742" ], "markers": "python_version < '3.8'", - "version": "==0.23" + "version": "==1.1.0" }, "isodate": { "hashes": [ @@ -340,6 +370,13 @@ ], "version": "==4.3.4" }, + "itsdangerous": { + "hashes": [ + "sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19", + "sha256:b12271b2047cb23eeb98c8b5622e2e5c5e9abd9784a153e9d8ef9cb4dd09d749" + ], + "version": "==1.1.0" + }, "jinja2": { "hashes": [ "sha256:74320bb91f31270f9551d46522e33af46a80c3d619f4a4bf42b3164d30b5911f", @@ -356,30 +393,34 @@ }, "lxml": { "hashes": [ - "sha256:02ca7bf899da57084041bb0f6095333e4d239948ad3169443f454add9f4e9cb4", - "sha256:096b82c5e0ea27ce9138bcbb205313343ee66a6e132f25c5ed67e2c8d960a1bc", - "sha256:0a920ff98cf1aac310470c644bc23b326402d3ef667ddafecb024e1713d485f1", - "sha256:17cae1730a782858a6e2758fd20dd0ef7567916c47757b694a06ffafdec20046", - "sha256:17e3950add54c882e032527795c625929613adbd2ce5162b94667334458b5a36", - "sha256:1f4f214337f6ee5825bf90a65d04d70aab05526c08191ab888cb5149501923c5", - "sha256:2e8f77db25b0a96af679e64ff9bf9dddb27d379c9900c3272f3041c4d1327c9d", - "sha256:4dffd405390a45ecb95ab5ab1c1b847553c18b0ef8ed01e10c1c8b1a76452916", - "sha256:6b899931a5648862c7b88c795eddff7588fb585e81cecce20f8d9da16eff96e0", - "sha256:726c17f3e0d7a7200718c9a890ccfeab391c9133e363a577a44717c85c71db27", - "sha256:760c12276fee05c36f95f8040180abc7fbebb9e5011447a97cdc289b5d6ab6fc", - "sha256:796685d3969815a633827c818863ee199440696b0961e200b011d79b9394bbe7", - "sha256:891fe897b49abb7db470c55664b198b1095e4943b9f82b7dcab317a19116cd38", - "sha256:a471628e20f03dcdfde00770eeaf9c77811f0c331c8805219ca7b87ac17576c5", - "sha256:a63b4fd3e2cabdcc9d918ed280bdde3e8e9641e04f3c59a2a3109644a07b9832", - "sha256:b0b84408d4eabc6de9dd1e1e0bc63e7731e890c0b378a62443e5741cfd0ae90a", - "sha256:be78485e5d5f3684e875dab60f40cddace2f5b2a8f7fede412358ab3214c3a6f", - "sha256:c27eaed872185f047bb7f7da2d21a7d8913457678c9a100a50db6da890bc28b9", - "sha256:c81cb40bff373ab7a7446d6bbca0190bccc5be3448b47b51d729e37799bb5692", - "sha256:d11874b3c33ee441059464711cd365b89fa1a9cf19ae75b0c189b01fbf735b84", - "sha256:e9c028b5897901361d81a4718d1db217b716424a0283afe9d6735fe0caf70f79", - "sha256:fe489d486cd00b739be826e8c1be188ddb74c7a1ca784d93d06fda882a6a1681" - ], - "version": "==4.4.1" + "sha256:00ac0d64949fef6b3693813fe636a2d56d97a5a49b5bbb86e4cc4cc50ebc9ea2", + "sha256:0571e607558665ed42e450d7bf0e2941d542c18e117b1ebbf0ba72f287ad841c", + "sha256:0e3f04a7615fdac0be5e18b2406529521d6dbdb0167d2a690ee328bef7807487", + "sha256:13cf89be53348d1c17b453867da68704802966c433b2bb4fa1f970daadd2ef70", + "sha256:217262fcf6a4c2e1c7cb1efa08bd9ebc432502abc6c255c4abab611e8be0d14d", + "sha256:223e544828f1955daaf4cefbb4853bc416b2ec3fd56d4f4204a8b17007c21250", + "sha256:277cb61fede2f95b9c61912fefb3d43fbd5f18bf18a14fae4911b67984486f5d", + "sha256:3213f753e8ae86c396e0e066866e64c6b04618e85c723b32ecb0909885211f74", + "sha256:4690984a4dee1033da0af6df0b7a6bde83f74e1c0c870623797cec77964de34d", + "sha256:4fcc472ef87f45c429d3b923b925704aa581f875d65bac80f8ab0c3296a63f78", + "sha256:61409bd745a265a742f2693e4600e4dbd45cc1daebe1d5fad6fcb22912d44145", + "sha256:678f1963f755c5d9f5f6968dded7b245dd1ece8cf53c1aa9d80e6734a8c7f41d", + "sha256:6c6d03549d4e2734133badb9ab1c05d9f0ef4bcd31d83e5d2b4747c85cfa21da", + "sha256:6e74d5f4d6ecd6942375c52ffcd35f4318a61a02328f6f1bd79fcb4ffedf969e", + "sha256:7b4fc7b1ecc987ca7aaf3f4f0e71bbfbd81aaabf87002558f5bc95da3a865bcd", + "sha256:7ed386a40e172ddf44c061ad74881d8622f791d9af0b6f5be20023029129bc85", + "sha256:8f54f0924d12c47a382c600c880770b5ebfc96c9fd94cf6f6bdc21caf6163ea7", + "sha256:ad9b81351fdc236bda538efa6879315448411a81186c836d4b80d6ca8217cdb9", + "sha256:bbd00e21ea17f7bcc58dccd13869d68441b32899e89cf6cfa90d624a9198ce85", + "sha256:c3c289762cc09735e2a8f8a49571d0e8b4f57ea831ea11558247b5bdea0ac4db", + "sha256:cf4650942de5e5685ad308e22bcafbccfe37c54aa7c0e30cd620c2ee5c93d336", + "sha256:cfcbc33c9c59c93776aa41ab02e55c288a042211708b72fdb518221cc803abc8", + "sha256:e301055deadfedbd80cf94f2f65ff23126b232b0d1fea28f332ce58137bcdb18", + "sha256:ebbfe24df7f7b5c6c7620702496b6419f6a9aa2fd7f005eb731cc80d7b4692b9", + "sha256:eff69ddbf3ad86375c344339371168640951c302450c5d3e9936e98d6459db06", + "sha256:f6ed60a62c5f1c44e789d2cf14009423cb1646b44a43e40a9cf6a21f077678a1" + ], + "version": "==4.4.2" }, "markupsafe": { "hashes": [ @@ -414,6 +455,14 @@ ], "version": "==1.1.1" }, + "marshmallow": { + "hashes": [ + "sha256:1a358beb89c2b4d5555272065a9533591a3eb02f1b854f3c4002d88d8f2a1ddb", + "sha256:eb97c42c5928b5720812c9268865fe863d4807bc1a8b48ddd7d5c9e1779a6af0" + ], + "markers": "python_version >= '3'", + "version": "==3.2.2" + }, "mccabe": { "hashes": [ "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", @@ -430,16 +479,17 @@ }, "more-itertools": { "hashes": [ - "sha256:409cd48d4db7052af495b09dec721011634af3753ae1ef92d2b32f73a745f832", - "sha256:92b8c4b06dac4f0611c0729b2f2ede52b2e1bac1ab48f089c7ddc12e26bb60c4" + "sha256:53ff73f186307d9c8ef17a9600309154a6ae27f25579e80af4db8f047ba14bc2", + "sha256:a0ea684c39bc4315ba7aae406596ef191fd84f873d2d2751f84d64e81a7a2d45" ], - "version": "==7.2.0" + "version": "==8.0.0" }, "mypy-extensions": { "hashes": [ - "sha256:a161e3b917053de87dbe469987e173e49fb454eca10ef28b48b384538cc11458" + "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d", + "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8" ], - "version": "==0.4.2" + "version": "==0.4.3" }, "ndg-httpsclient": { "hashes": [ @@ -451,9 +501,17 @@ }, "networkx": { "hashes": [ - "sha256:8311ddef63cf5c5c5e7c1d0212dd141d9a1fe3f474915281b73597ed5f1d4e3d" + "sha256:cdfbf698749a5014bf2ed9db4a07a5295df1d3a53bf80bf3cbd61edf9df05fa1", + "sha256:f8f4ff0b6f96e4f9b16af6b84622597b5334bf9cae8cf9b2e42e7985d5c95c64" ], - "version": "==2.3" + "version": "==2.4" + }, + "owlrl": { + "hashes": [ + "sha256:2ad753f5ba6d1fe2d88bf36f427df31553f2eaa0283692e3cd06cab20ac8aec3", + "sha256:efdebe76cf9ad148f316a9ae92e898e12b3b3690bd90218c898a2b676955b266" + ], + "version": "==5.2.1" }, "packaging": { "hashes": [ @@ -478,10 +536,10 @@ }, "pluggy": { "hashes": [ - "sha256:0db4b7601aae1d35b4a033282da476845aa19185c1e6964b25cf324b5e4ec3e6", - "sha256:fa5fa1622fa6dd5c030e9cad086fa19ef6a0cf6d7a2d12318e10cb49d6d68f34" + "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0", + "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d" ], - "version": "==0.13.0" + "version": "==0.13.1" }, "prov": { "hashes": [ @@ -492,17 +550,19 @@ }, "psutil": { "hashes": [ - "sha256:028a1ec3c6197eadd11e7b46e8cc2f0720dc18ac6d7aabdb8e8c0d6c9704f000", - "sha256:503e4b20fa9d3342bcf58191bbc20a4a5ef79ca7df8972e6197cc14c5513e73d", - "sha256:863a85c1c0a5103a12c05a35e59d336e1d665747e531256e061213e2e90f63f3", - "sha256:954f782608bfef9ae9f78e660e065bd8ffcfaea780f9f2c8a133bb7cb9e826d7", - "sha256:b6e08f965a305cd84c2d07409bc16fbef4417d67b70c53b299116c5b895e3f45", - "sha256:bc96d437dfbb8865fc8828cf363450001cb04056bbdcdd6fc152c436c8a74c61", - "sha256:cf49178021075d47c61c03c0229ac0c60d5e2830f8cab19e2d88e579b18cdb76", - "sha256:d5350cb66690915d60f8b233180f1e49938756fb2d501c93c44f8fb5b970cc63", - "sha256:eba238cf1989dfff7d483c029acb0ac4fcbfc15de295d682901f0e2497e6781a" + "sha256:094f899ac3ef72422b7e00411b4ed174e3c5a2e04c267db6643937ddba67a05b", + "sha256:10b7f75cc8bd676cfc6fa40cd7d5c25b3f45a0e06d43becd7c2d2871cbb5e806", + "sha256:1b1575240ca9a90b437e5a40db662acd87bbf181f6aa02f0204978737b913c6b", + "sha256:21231ef1c1a89728e29b98a885b8e0a8e00d09018f6da5cdc1f43f988471a995", + "sha256:28f771129bfee9fc6b63d83a15d857663bbdcae3828e1cb926e91320a9b5b5cd", + "sha256:70387772f84fa5c3bb6a106915a2445e20ac8f9821c5914d7cbde148f4d7ff73", + "sha256:b560f5cd86cf8df7bcd258a851ca1ad98f0d5b8b98748e877a0aec4e9032b465", + "sha256:b74b43fecce384a57094a83d2778cdfc2e2d9a6afaadd1ebecb2e75e0d34e10d", + "sha256:e85f727ffb21539849e6012f47b12f6dd4c44965e56591d8dec6e8bc9ab96f4a", + "sha256:fd2e09bb593ad9bdd7429e779699d2d47c1268cbde4dda95fcd1bd17544a0217", + "sha256:ffad8eb2ac614518bbe3c0b8eb9dffdb3a8d2e3a7d5da51c5b974fb723a5c5aa" ], - "version": "==5.6.3" + "version": "==5.6.7" }, "py": { "hashes": [ @@ -513,10 +573,10 @@ }, "pyasn1": { "hashes": [ - "sha256:62cdade8b5530f0b185e09855dd422bc05c0bbff6b72ff61381c09dac7befd8c", - "sha256:a9495356ca1d66ed197a0f72b41eb1823cf7ea8b5bd07191673e8147aecf8604" + "sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d", + "sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba" ], - "version": "==0.4.7" + "version": "==0.4.8" }, "pycodestyle": { "hashes": [ @@ -547,10 +607,10 @@ }, "pygments": { "hashes": [ - "sha256:71e430bc85c88a430f000ac1d9b331d2407f681d6f6aec95e8bcfbc3df5b0127", - "sha256:881c4c157e45f30af185c1ffe8d549d48ac9127433f2c380c24b84572ad66297" + "sha256:2a3fe295e54a20164a9df49c75fa58526d3be48e14aceba6d6b1e8ac0bfd6f1b", + "sha256:98c8aa5a9f778fcd1026a17361ddaf7330d1b7c62ae97c3bb0ae73e0b9b6b0fe" ], - "version": "==2.4.2" + "version": "==2.5.2" }, "pyld": { "hashes": [ @@ -560,24 +620,31 @@ }, "pyopenssl": { "hashes": [ - "sha256:aeca66338f6de19d1aa46ed634c3b9ae519a64b458f8468aec688e7e3c20f200", - "sha256:c727930ad54b10fc157015014b666f2d8b41f70c0d03e83ab67624fd3dd5d1e6" + "sha256:621880965a720b8ece2f1b2f54ea2071966ab00e2970ad2ce11d596102063504", + "sha256:9a24494b2602aaf402be5c9e30a0b82d4a5c67528fe8fb475e3f3bc00dd69507" ], - "version": "==19.0.0" + "version": "==19.1.0" }, "pyparsing": { "hashes": [ - "sha256:6f98a7b9397e206d78cc01df10131398f1c8b8510a2f4d97d9abd82e1aacdd80", - "sha256:d9338df12903bbf5d65a0e4e87c2161968b10d2e489652bb47001d82a9b028b4" + "sha256:20f995ecd72f2a1f4bf6b072b63b22e2eb457836601e76d6e5dfcd75436acc1f", + "sha256:4ca62001be367f01bd3e92ecbb79070272a9d4964dce6a48a82ff0b8bc7e683a" ], - "version": "==2.4.2" + "version": "==2.4.5" + }, + "pyshacl": { + "hashes": [ + "sha256:74739aa88dcdb161849f769d28e3a14ed0b01eb8256a21cc2941a27dc80e70af", + "sha256:d551c0e5842f30151b3e398407f5ffc8250e602add925f0bb31bce5771419169" + ], + "version": "==0.11.3.post1" }, "pytest": { "hashes": [ - "sha256:7e4800063ccfc306a53c461442526c5571e1462f61583506ce97e4da6a1d88c8", - "sha256:ca563435f4941d0cb34767301c27bc65c510cb82e90b9ecf9cb52dc2c63caaa0" + "sha256:63344a2e3bce2e4d522fd62b4fdebb647c019f1f9e4ca075debbd13219db4418", + "sha256:f67403f33b2b1d25a6756184077394167fe5e2f9d8bdaab30707d19ccec35427" ], - "version": "==5.2.1" + "version": "==5.3.1" }, "pytest-cache": { "hashes": [ @@ -607,10 +674,10 @@ }, "pytest-runner": { "hashes": [ - "sha256:25a013c8d84f0ca60bb01bd11913a3bcab420f601f0f236de4423074af656e7a", - "sha256:d04243fbf29a3b574f18f1bcff2a07f505db5daede82f706f2e32728f77d3f4d" + "sha256:5534b08b133ef9a5e2c22c7886a8f8508c95bb0b0bdc6cc13214f269c3c70d51", + "sha256:96c7e73ead7b93e388c5d614770d2bae6526efd997757d3543fe17b557a0942b" ], - "version": "==5.1" + "version": "==5.2" }, "pytest-yapf": { "hashes": [ @@ -622,10 +689,10 @@ }, "python-dateutil": { "hashes": [ - "sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb", - "sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e" + "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c", + "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a" ], - "version": "==2.8.0" + "version": "==2.8.1" }, "python-editor": { "hashes": [ @@ -673,6 +740,13 @@ ], "version": "==0.4.0" }, + "redis": { + "hashes": [ + "sha256:3613daad9ce5951e426f460deddd5caf469e08a3af633e9578fc77d362becf62", + "sha256:8d0fc278d3f5e1249967cba2eb4a5632d19e45ce5c09442b8422d15ee2c22cc2" + ], + "version": "==3.3.11" + }, "renku": { "editable": true, "extras": [ @@ -696,10 +770,10 @@ }, "responses": { "hashes": [ - "sha256:502d9c0c8008439cfcdef7e251f507fcfdd503b56e8c0c87c3c3e3393953f790", - "sha256:97193c0183d63fba8cd3a041c75464e4b09ea0aff6328800d1546598567dde0b" + "sha256:46d4e546a19fc6106bc7e804edd4551ef04690405e41e7e750ebc295d042623b", + "sha256:93b1e0f2f960c0f3304ca4436856241d64c33683ef441431b9caf1d05d9d9e23" ], - "version": "==0.10.6" + "version": "==0.10.7" }, "ruamel.yaml": { "hashes": [ @@ -753,10 +827,10 @@ }, "sentry-sdk": { "hashes": [ - "sha256:15e51e74b924180c98bcd636cb4634945b0a99a124d50b433c3a9dc6a582e8db", - "sha256:1d6a2ee908ec6d8f96c27d78bc39e203df4d586d287c233140af7d8d1aca108a" + "sha256:a7c2c8d3f53b6b57454830cd6a4b73d272f1ba91952f59e6545b3cf885f3c22f", + "sha256:bfc486af718c268cf49ff43d6334ed4db7333ace420240b630acdd8f8a3a8f60" ], - "version": "==0.12.3" + "version": "==0.13.4" }, "setuptools-scm": { "hashes": [ @@ -781,10 +855,10 @@ }, "six": { "hashes": [ - "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", - "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" + "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd", + "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66" ], - "version": "==1.12.0" + "version": "==1.13.0" }, "smmap2": { "hashes": [ @@ -800,12 +874,19 @@ ], "version": "==2.0.0" }, + "sortedcontainers": { + "hashes": [ + "sha256:974e9a32f56b17c1bac2aebd9dcf197f3eb9cd30553c5852a3187ad162e1a03a", + "sha256:d9e96492dd51fae31e60837736b38fe42a187b5404c16606ff7ee7cd582d4c60" + ], + "version": "==2.1.0" + }, "sphinx": { "hashes": [ - "sha256:0d586b0f8c2fc3cc6559c5e8fd6124628110514fda0e5d7c82e682d749d2e845", - "sha256:839a3ed6f6b092bb60f492024489cc9e6991360fb9f52ed6361acd510d261069" + "sha256:31088dfb95359384b1005619827eaee3056243798c62724fd3fa4b84ee4d71bd", + "sha256:52286a0b9d7caa31efee301ec4300dbdab23c3b05da1c9024b4e84896fb73d79" ], - "version": "==2.2.0" + "version": "==2.2.1" }, "sphinxcontrib-applehelp": { "hashes": [ @@ -859,9 +940,9 @@ }, "tabulate": { "hashes": [ - "sha256:d0097023658d4dea848d6ae73af84532d1e86617ac0925d1adf1dd903985dac3" + "sha256:5470cc6687a091c7042cee89b2946d9235fe9f6d49c193a4ae2ac7bf386737c8" ], - "version": "==0.8.5" + "version": "==0.8.6" }, "toml": { "hashes": [ @@ -872,18 +953,18 @@ }, "tqdm": { "hashes": [ - "sha256:abc25d0ce2397d070ef07d8c7e706aede7920da163c64997585d42d3537ece3d", - "sha256:dd3fcca8488bb1d416aa7469d2f277902f26260c45aa86b667b074cd44b3b115" + "sha256:156a0565f09d1f0ef8242932a0e1302462c93827a87ba7b4423d90f01befe94c", + "sha256:c0ffb55959ea5f3eaeece8d2db0651ba9ced9c72f40a6cce3419330256234289" ], - "version": "==4.36.1" + "version": "==4.40.0" }, "typing-extensions": { "hashes": [ - "sha256:2ed632b30bb54fc3941c382decfd0ee4148f5c591651c9272473fea2c6397d95", - "sha256:b1edbbf0652660e32ae780ac9433f4231e7339c7f9a8057d0f042fcbcea49b87", - "sha256:d8179012ec2c620d3791ca6fe2bf7979d979acdbef1fca0bc56b37411db682ed" + "sha256:091ecc894d5e908ac75209f10d5b4f118fbdb2eb1ede6a63544054bb1edb41f2", + "sha256:910f4656f54de5993ad9304959ce9bb903f90aadc7c67a0bef07e678014e892d", + "sha256:cf8b63fedea4d89bab840ecbb93e75578af28f76f66c35889bd7065f5af88575" ], - "version": "==3.7.4" + "version": "==3.7.4.1" }, "unify": { "hashes": [ @@ -899,10 +980,10 @@ }, "urllib3": { "hashes": [ - "sha256:3de946ffbed6e6746608990594d08faac602528ac7015ac28d33cee6a45b7398", - "sha256:9a107b99a5393caf59c7aa3c1249c16e6879447533d0887f4336dde834c7be86" + "sha256:a8a318824cc77d1fd4b2bec2ded92646630d7fe8619497b142c84a9e6f5a7293", + "sha256:f3c5fd51747d450d4dcf6f923c81f78f811aab8205fda64b0aba34a4e48b0745" ], - "version": "==1.25.6" + "version": "==1.25.7" }, "wcwidth": { "hashes": [ @@ -911,6 +992,14 @@ ], "version": "==0.1.7" }, + "webargs": { + "hashes": [ + "sha256:3beca296598067cec24a0b6f91c0afcc19b6e3c4d84ab026b931669628bb47b4", + "sha256:3f9dc15de183d356c9a0acc159c100ea0506c0c240c1e6f1d8b308c5fed4dbbd", + "sha256:fa4ad3ad9b38bedd26c619264fdc50d7ae014b49186736bca851e5b5228f2a1b" + ], + "version": "==5.5.2" + }, "werkzeug": { "hashes": [ "sha256:7280924747b5733b246fe23972186c6b348f9ae29724135a6dfc1e53cea433e7", diff --git a/conftest.py b/conftest.py index a0e4fa49bd..63f880b524 100644 --- a/conftest.py +++ b/conftest.py @@ -25,8 +25,10 @@ import tempfile import time import urllib +import uuid from pathlib import Path +import fakeredis import pytest import responses import yaml @@ -510,3 +512,176 @@ def remote_project(data_repository, directory_tree): assert 0 == result.exit_code yield runner, project_path + + +@pytest.fixture(scope='function') +def dummy_datapack(): + """Creates dummy data folder.""" + temp_dir = tempfile.TemporaryDirectory() + + data_file_txt = Path(temp_dir.name) / Path('file.txt') + data_file_txt.write_text('my awesome data') + + data_file_csv = Path(temp_dir.name) / Path('file.csv') + data_file_csv.write_text('more,awesome,data') + + yield temp_dir + + +@pytest.fixture(scope='function') +def datapack_zip(dummy_datapack): + """Returns dummy data folder as a zip archive.""" + from renku.core.utils.contexts import chdir + workspace_dir = tempfile.TemporaryDirectory() + with chdir(workspace_dir.name): + shutil.make_archive('datapack', 'zip', dummy_datapack.name) + + yield Path(workspace_dir.name) / 'datapack.zip' + + +@pytest.fixture(scope='function') +def datapack_tar(dummy_datapack): + """Returns dummy data folder as a tar archive.""" + from renku.core.utils.contexts import chdir + workspace_dir = tempfile.TemporaryDirectory() + with chdir(workspace_dir.name): + shutil.make_archive('datapack', 'tar', dummy_datapack.name) + + yield Path(workspace_dir.name) / 'datapack.tar' + + +@pytest.fixture(scope='function') +def mock_redis(monkeypatch): + """Monkey patch service cache with mocked redis.""" + from renku.service.cache import ServiceCache + with monkeypatch.context() as m: + m.setattr(ServiceCache, 'cache', fakeredis.FakeRedis()) + yield + + +@pytest.fixture(scope='function') +def svc_client(mock_redis): + """Renku service client.""" + from renku.service.entrypoint import create_app + + flask_app = create_app() + + testing_client = flask_app.test_client() + testing_client.testing = True + + ctx = flask_app.app_context() + ctx.push() + + yield testing_client + + ctx.pop() + + +@pytest.fixture(scope='function') +def svc_client_with_repo(svc_client, mock_redis): + """Renku service remote repository.""" + remote_url = 'https://dev.renku.ch/gitlab/contact/integration-tests' + headers = { + 'Content-Type': 'application/json', + 'Renku-User-Id': 'b4b4de0eda0f471ab82702bd5c367fa7', + 'Renku-User-FullName': 'Just Sam', + 'Renku-User-Email': 'contact@justsam.io', + 'Authorization': 'Bearer LkoLiyLqnhMCAa4or5qa', + } + + payload = {'git_url': remote_url} + + response = svc_client.post( + '/cache/project-clone', + data=json.dumps(payload), + headers=headers, + ) + + assert response + assert 'result' in response.json + assert 'error' not in response.json + project_id = response.json['result']['project_id'] + assert isinstance(uuid.UUID(project_id), uuid.UUID) + + yield svc_client, headers, project_id + + +@pytest.fixture( + params=[ + { + 'url': '/cache/files-list', + 'allowed_method': 'GET', + 'headers': { + 'Content-Type': 'application/json', + 'accept': 'application/json', + } + }, + { + 'url': '/cache/files-upload', + 'allowed_method': 'POST', + 'headers': {} + }, + { + 'url': '/cache/project-clone', + 'allowed_method': 'POST', + 'headers': { + 'Content-Type': 'application/json', + 'accept': 'application/json', + } + }, + { + 'url': '/cache/project-list', + 'allowed_method': 'GET', + 'headers': { + 'Content-Type': 'application/json', + 'accept': 'application/json', + } + }, + { + 'url': '/datasets/add', + 'allowed_method': 'POST', + 'headers': { + 'Content-Type': 'application/json', + 'accept': 'application/json', + } + }, + { + 'url': '/datasets/create', + 'allowed_method': 'POST', + 'headers': { + 'Content-Type': 'application/json', + 'accept': 'application/json', + } + }, + { + 'url': '/datasets/files-list', + 'allowed_method': 'GET', + 'headers': { + 'Content-Type': 'application/json', + 'accept': 'application/json', + } + }, + { + 'url': '/datasets/list', + 'allowed_method': 'GET', + 'headers': { + 'Content-Type': 'application/json', + 'accept': 'application/json', + } + }, + ] +) +def service_allowed_endpoint(request, svc_client, mock_redis): + """Ensure allowed methods and correct headers.""" + methods = { + 'GET': svc_client.get, + 'POST': svc_client.post, + 'HEAD': svc_client.head, + 'PUT': svc_client.put, + 'DELETE': svc_client.delete, + 'OPTIONS': svc_client.options, + 'TRACE': svc_client.trace, + 'PATCH': svc_client.patch, + } + + yield methods, request.param, svc_client diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000000..96436d1b24 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,13 @@ +version: '3' + +services: + redis: + image: redis:5.0.3-alpine + ports: + - "6379:6379" + + renku-svc: + image: renku-svc:latest + env_file: .env + ports: + - "8080:8080" diff --git a/renku/cli/__init__.py b/renku/cli/__init__.py index e6dfe7d750..1021fcb38d 100644 --- a/renku/cli/__init__.py +++ b/renku/cli/__init__.py @@ -91,7 +91,7 @@ option_use_external_storage from renku.core.commands.version import check_version, print_version from renku.core.management.client import LocalClient -from renku.core.management.config import ConfigManagerMixin, RENKU_HOME +from renku.core.management.config import RENKU_HOME, ConfigManagerMixin from renku.core.management.repository import default_path #: Monkeypatch Click application. diff --git a/renku/core/commands/client.py b/renku/core/commands/client.py index 12d103fae0..f9eec0d0e2 100644 --- a/renku/core/commands/client.py +++ b/renku/core/commands/client.py @@ -25,6 +25,8 @@ import yaml from renku.core.management import LocalClient +from renku.core.management.config import RENKU_HOME +from renku.core.management.repository import default_path from .git import get_git_isolation @@ -63,8 +65,17 @@ def pass_local_client( ) def new_func(*args, **kwargs): - ctx = click.get_current_context() - client = ctx.ensure_object(LocalClient) + ctx = click.get_current_context(silent=True) + if not ctx: + client = LocalClient( + path=default_path(), + renku_home=RENKU_HOME, + use_external_storage=True, + ) + ctx = click.Context(click.Command(method)) + else: + client = ctx.ensure_object(LocalClient) + stack = contextlib.ExitStack() # Handle --isolation option: @@ -85,8 +96,11 @@ def new_func(*args, **kwargs): if lock or (lock is None and commit): stack.enter_context(client.lock) - with stack: - result = ctx.invoke(method, client, *args, **kwargs) + result = None + if ctx: + with stack: + result = ctx.invoke(method, client, *args, **kwargs) + return result return functools.update_wrapper(new_func, method) diff --git a/renku/core/commands/clone.py b/renku/core/commands/clone.py index d3805640a5..003f2a792a 100644 --- a/renku/core/commands/clone.py +++ b/renku/core/commands/clone.py @@ -29,7 +29,11 @@ def renku_clone( path=None, install_githooks=True, skip_smudge=True, - progress=None + recursive=True, + depth=None, + progress=None, + config=None, + raise_git_except=False ): """Clone Renku project repo, install Git hooks and LFS.""" install_lfs = client.use_external_storage @@ -39,5 +43,9 @@ def renku_clone( install_githooks=install_githooks, install_lfs=install_lfs, skip_smudge=skip_smudge, - progress=progress + recursive=recursive, + depth=depth, + progress=progress, + config=config, + raise_git_except=raise_git_except, ) diff --git a/renku/core/commands/dataset.py b/renku/core/commands/dataset.py index 6604b2838a..ef7734c2ec 100644 --- a/renku/core/commands/dataset.py +++ b/renku/core/commands/dataset.py @@ -146,7 +146,7 @@ def add_file( destination='', ref=None, with_metadata=None, - urlscontext=contextlib.nullcontext + urlscontext=contextlib.nullcontext, ): """Add data file to a dataset.""" add_to_dataset( diff --git a/renku/core/management/clone.py b/renku/core/management/clone.py index 5c4f900587..4de5f8f86c 100644 --- a/renku/core/management/clone.py +++ b/renku/core/management/clone.py @@ -18,6 +18,7 @@ """Clone a Renku repo along with all Renku-specific initializations.""" import os +from pathlib import Path from git import GitCommandError, Repo @@ -33,23 +34,46 @@ def clone( skip_smudge=True, recursive=True, depth=None, - progress=None + progress=None, + config=None, + raise_git_except=False, ): """Clone Renku project repo, install Git hooks and LFS.""" from renku.core.management.client import LocalClient path = path or '.' + + if isinstance(path, Path): + path = str(path) + # Clone the project if skip_smudge: os.environ['GIT_LFS_SKIP_SMUDGE'] = '1' + try: repo = Repo.clone_from( url, path, recursive=recursive, depth=depth, progress=progress ) except GitCommandError as e: - raise errors.GitError( - 'Cannot clone remote Renku project: {}'.format(url) - ) from e + if not raise_git_except: + raise errors.GitError( + 'Cannot clone remote Renku project: {}'.format(url) + ) from e + + raise e + + if config: + config_writer = repo.config_writer() + + for key, value in config.items(): + key_path = key.split('.') + if len(key_path) != 2: + raise errors.GitError( + 'Cannot write to config path: {0}'.format(key) + ) + config_writer.set_value(key_path[0], key_path[1], value) + + config_writer.release() client = LocalClient(path) diff --git a/renku/core/management/repository.py b/renku/core/management/repository.py index a937390738..114081ed40 100644 --- a/renku/core/management/repository.py +++ b/renku/core/management/repository.py @@ -47,13 +47,18 @@ def default_path(): return '.' +def path_converter(path): + """Converter for path in PathMixin.""" + return Path(path).resolve() + + @attr.s class PathMixin: """Define a default path attribute.""" path = attr.ib( default=default_path, - converter=lambda arg: Path(arg).resolve().absolute(), + converter=path_converter, ) @path.validator diff --git a/renku/core/utils/contexts.py b/renku/core/utils/contexts.py index 77de0bc61b..06131ade45 100644 --- a/renku/core/utils/contexts.py +++ b/renku/core/utils/contexts.py @@ -26,6 +26,9 @@ @contextlib.contextmanager def chdir(path): """Change the current working directory.""" + if isinstance(path, Path): + path = str(path) + cwd = os.getcwd() os.chdir(path) try: diff --git a/renku/service/.env-example b/renku/service/.env-example new file mode 100644 index 0000000000..45f635b6c3 --- /dev/null +++ b/renku/service/.env-example @@ -0,0 +1,7 @@ +REDIS_HOST=redis +REDIS_PORT=6379 +REDIS_DATABASE=0 +REDIS_PASSWORD= + +CACHE_DIR= +PROJECT_CLONE_DEPTH_DEFAULT=1 diff --git a/renku/service/__init__.py b/renku/service/__init__.py new file mode 100644 index 0000000000..1928b35350 --- /dev/null +++ b/renku/service/__init__.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service.""" diff --git a/renku/service/cache/__init__.py b/renku/service/cache/__init__.py new file mode 100644 index 0000000000..ca18ae78ec --- /dev/null +++ b/renku/service/cache/__init__.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service cache management for files.""" +from renku.service.cache.files import FileManagementCache +from renku.service.cache.projects import ProjectManagementCache + + +class ServiceCache(FileManagementCache, ProjectManagementCache): + """Service cache manager.""" + + pass diff --git a/renku/service/cache/base.py b/renku/service/cache/base.py new file mode 100644 index 0000000000..c38f3f8d79 --- /dev/null +++ b/renku/service/cache/base.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service cache management.""" +import json + +import redis +from redis import RedisError + +from renku.service.cache.config import REDIS_DATABASE, REDIS_HOST, \ + REDIS_PASSWORD, REDIS_PORT + + +class BaseCache: + """Cache management.""" + + cache = redis.Redis( + host=REDIS_HOST, + port=REDIS_PORT, + db=REDIS_DATABASE, + password=REDIS_PASSWORD + ) + + def set_record(self, name, key, value): + """Insert a record to hash set.""" + if isinstance(value, dict): + value = json.dumps(value) + + self.cache.hset(name, key, value) + + def invalidate_key(self, name, key): + """Invalidate cache `key` in users hash set.""" + try: + self.cache.hdel(name, key) + except RedisError: + pass + + def get_record(self, name, key): + """Return record values from hash set.""" + result = self.cache.hget(name, key) + if result: + return json.loads(result.decode('utf-8')) + + def get_all_records(self, name): + """Return all record values from hash set.""" + return [ + json.loads(record.decode('utf-8')) + for record in self.cache.hgetall(name).values() + ] diff --git a/renku/service/cache/config.py b/renku/service/cache/config.py new file mode 100644 index 0000000000..7afb2d6b68 --- /dev/null +++ b/renku/service/cache/config.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service cache configuration.""" +import os + +REDIS_HOST = os.getenv('REDIS_HOST', '0.0.0.0') +REDIS_PORT = int(os.getenv('REDIS_PORT', 6379)) +REDIS_DATABASE = int(os.getenv('REDIS_DATABASE', 0)) +REDIS_PASSWORD = os.getenv('REDIS_PASSWORD') diff --git a/renku/service/cache/files.py b/renku/service/cache/files.py new file mode 100644 index 0000000000..ab8240c79e --- /dev/null +++ b/renku/service/cache/files.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service files cache management.""" +from renku.service.cache.base import BaseCache + + +class FileManagementCache(BaseCache): + """File management cache.""" + + FILES_SUFFIX = 'files' + + def files_cache_key(self, user): + """Construct cache key based on user and files suffix.""" + return '{0}_{1}'.format(user, self.FILES_SUFFIX) + + def set_file(self, user, file_id, metadata): + """Cache file metadata under user hash set.""" + self.set_record(self.files_cache_key(user), file_id, metadata) + + def set_files(self, user, files): + """Cache a list of metadata files under user hash set.""" + for file_ in files: + self.set_file(user, file_['file_id'], file_) + + def get_files(self, user): + """Get all user cached files.""" + return self.get_all_records(self.files_cache_key(user)) + + def get_file(self, user, file_id): + """Get user cached file.""" + result = self.get_record(self.files_cache_key(user), file_id) + return result + + def invalidate_file(self, user, file_id): + """Remove file record from hash set.""" + self.invalidate_key(self.files_cache_key(user), file_id) diff --git a/renku/service/cache/projects.py b/renku/service/cache/projects.py new file mode 100644 index 0000000000..21f10e9a8a --- /dev/null +++ b/renku/service/cache/projects.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service project cache management.""" +from renku.service.cache.base import BaseCache + + +class ProjectManagementCache(BaseCache): + """Project management cache.""" + + PROJECTS_SUFFIX = 'projects' + + def projects_cache_key(self, user): + """Construct cache key based on user and projects suffix.""" + return '{0}_{1}'.format(user, self.PROJECTS_SUFFIX) + + def set_project(self, user, project_id, metadata): + """Cache project metadata under user hash set.""" + self.set_record(self.projects_cache_key(user), project_id, metadata) + + def get_projects(self, user): + """Get all user cache projects.""" + return self.get_all_records(self.projects_cache_key(user)) + + def get_project(self, user, project_id): + """Get user cached project.""" + result = self.get_record(self.projects_cache_key(user), project_id) + return result + + def invalidate_project(self, user, project_id): + """Remove project record from hash set.""" + self.invalidate_key(self.projects_cache_key(user), project_id) diff --git a/renku/service/config.py b/renku/service/config.py new file mode 100644 index 0000000000..774f2c775a --- /dev/null +++ b/renku/service/config.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service config.""" +import os +import tempfile +from pathlib import Path + +GIT_ACCESS_DENIED_ERROR_CODE = -32000 +GIT_UNKNOWN_ERROR_CODE = -32001 + +RENKU_EXCEPTION_ERROR_CODE = -32100 +REDIS_EXCEPTION_ERROR_CODE = -32200 + +INVALID_HEADERS_ERROR_CODE = -32601 +INVALID_PARAMS_ERROR_CODE = -32602 +INTERNAL_FAILURE_ERROR_CODE = -32603 + +SERVICE_NAME = 'Renku Service' +OPENAPI_VERSION = '2.0' +API_VERSION = 'v1' + +SWAGGER_URL = '/api/docs' +API_SPEC_URL = os.getenv( + 'RENKU_SVC_SWAGGER_URL', '/api/{0}/spec'.format(API_VERSION) +) + +PROJECT_CLONE_DEPTH_DEFAULT = int(os.getenv('PROJECT_CLONE_DEPTH_DEFAULT', 1)) + +CACHE_DIR = os.getenv('CACHE_DIR', tempfile.TemporaryDirectory().name) +CACHE_UPLOADS_PATH = Path(CACHE_DIR) / Path('uploads') +CACHE_UPLOADS_PATH.mkdir(parents=True, exist_ok=True) + +CACHE_PROJECTS_PATH = Path(CACHE_DIR) / Path('projects') +CACHE_PROJECTS_PATH.mkdir(parents=True, exist_ok=True) + +TAR_ARCHIVE_CONTENT_TYPE = 'application/x-tar' +ZIP_ARCHIVE_CONTENT_TYPE = 'application/zip' + +SUPPORTED_ARCHIVES = [ + TAR_ARCHIVE_CONTENT_TYPE, + ZIP_ARCHIVE_CONTENT_TYPE, +] diff --git a/renku/service/entrypoint.py b/renku/service/entrypoint.py new file mode 100644 index 0000000000..7f645c9e90 --- /dev/null +++ b/renku/service/entrypoint.py @@ -0,0 +1,102 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service entry point.""" +import os +import uuid + +from apispec import APISpec +from apispec.ext.marshmallow import MarshmallowPlugin +from flask import Flask +from flask_apispec import FlaskApiSpec +from flask_swagger_ui import get_swaggerui_blueprint + +from renku.service.cache import ServiceCache +from renku.service.config import API_SPEC_URL, API_VERSION, CACHE_DIR, \ + CACHE_PROJECTS_PATH, CACHE_UPLOADS_PATH, OPENAPI_VERSION, SERVICE_NAME, \ + SWAGGER_URL +from renku.service.views.cache import CACHE_BLUEPRINT_TAG, cache_blueprint, \ + list_projects_view, list_uploaded_files_view, project_clone, \ + upload_file_view +from renku.service.views.datasets import DATASET_BLUEPRINT_TAG, \ + add_file_to_dataset_view, create_dataset_view, dataset_blueprint, \ + list_dataset_files_view, list_datasets_view + + +def make_cache(): + """Create cache structure.""" + sub_dirs = [CACHE_UPLOADS_PATH, CACHE_PROJECTS_PATH] + + for subdir in sub_dirs: + if not subdir.exists(): + subdir.mkdir() + + return ServiceCache() + + +def create_app(): + """Creates a Flask app with necessary configuration.""" + app = Flask(__name__) + app.secret_key = os.getenv('RENKU_SVC_SERVICE_KEY', uuid.uuid4().hex) + + app.config['UPLOAD_FOLDER'] = CACHE_DIR + app.config['MAX_CONTENT_LENGTH'] = 16 * 1024 * 1024 + + cache = make_cache() + app.config['cache'] = cache + + build_routes(app) + return app + + +def build_routes(app): + """Register routes to given app instance.""" + app.config.update({ + 'APISPEC_SPEC': + APISpec( + title=SERVICE_NAME, + openapi_version=OPENAPI_VERSION, + version=API_VERSION, + plugins=[MarshmallowPlugin()], + ), + 'APISPEC_SWAGGER_URL': API_SPEC_URL, + }) + app.register_blueprint(cache_blueprint) + app.register_blueprint(dataset_blueprint) + + swaggerui_blueprint = get_swaggerui_blueprint( + SWAGGER_URL, API_SPEC_URL, config={'app_name': 'Renku Service'} + ) + app.register_blueprint(swaggerui_blueprint, url_prefix=SWAGGER_URL) + + docs = FlaskApiSpec(app) + + docs.register(upload_file_view, blueprint=CACHE_BLUEPRINT_TAG) + docs.register(list_uploaded_files_view, blueprint=CACHE_BLUEPRINT_TAG) + docs.register(project_clone, blueprint=CACHE_BLUEPRINT_TAG) + docs.register(list_projects_view, blueprint=CACHE_BLUEPRINT_TAG) + + docs.register(create_dataset_view, blueprint=DATASET_BLUEPRINT_TAG) + docs.register(add_file_to_dataset_view, blueprint=DATASET_BLUEPRINT_TAG) + docs.register(list_datasets_view, blueprint=DATASET_BLUEPRINT_TAG) + docs.register(list_dataset_files_view, blueprint=DATASET_BLUEPRINT_TAG) + + +app = create_app() + +if __name__ == '__main__': + app.run() diff --git a/renku/service/serializers/__init__.py b/renku/service/serializers/__init__.py new file mode 100644 index 0000000000..362f6221d1 --- /dev/null +++ b/renku/service/serializers/__init__.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service serializers.""" diff --git a/renku/service/serializers/cache.py b/renku/service/serializers/cache.py new file mode 100644 index 0000000000..c42415245e --- /dev/null +++ b/renku/service/serializers/cache.py @@ -0,0 +1,171 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service cache serializers.""" +import time +import uuid +from urllib.parse import urlparse + +from marshmallow import Schema, ValidationError, fields, post_load, pre_load, \ + validates +from werkzeug.utils import secure_filename + +from renku.core.errors import ConfigurationError +from renku.core.models.git import GitURL +from renku.service.config import PROJECT_CLONE_DEPTH_DEFAULT +from renku.service.serializers.rpc import JsonRPCResponse + + +def extract_file(request): + """Extract file from Flask request. + + :raises: `ValidationError` + """ + files = request.files + if 'file' not in files: + raise ValidationError('missing key: file') + + file = files['file'] + if file and not file.filename: + raise ValidationError('wrong filename: {0}'.format(file.filename)) + + if file: + file.filename = secure_filename(file.filename) + return file + + +class FileUploadRequest(Schema): + """Request schema for file upload.""" + + override_existing = fields.Boolean(missing=False) + unpack_archive = fields.Boolean(missing=False) + + +class FileUploadContext(Schema): + """Context schema for file upload.""" + + file_id = fields.String(missing=lambda: uuid.uuid4().hex) + + # measured in ms + timestamp = fields.Integer(missing=time.time() * 1e+3) + + content_type = fields.String(missing='unknown') + file_name = fields.String(required=True) + + # measured in bytes (comes from stat() - st_size) + file_size = fields.Integer(required=True) + + relative_path = fields.String(required=True) + is_archive = fields.Boolean(missing=False) + unpack_archive = fields.Boolean(missing=False) + + +class FileUploadResponse(Schema): + """Response schema for file upload.""" + + files = fields.List(fields.Nested(FileUploadContext), required=True) + + +class FileUploadResponseRPC(JsonRPCResponse): + """RPC response schema for file upload response.""" + + result = fields.Nested(FileUploadResponse) + + +class FileListResponse(Schema): + """Response schema for files listing.""" + + files = fields.List(fields.Nested(FileUploadContext), required=True) + + +class FileListResponseRPC(JsonRPCResponse): + """RPC response schema for files listing.""" + + result = fields.Nested(FileListResponse) + + +class ProjectCloneRequest(Schema): + """Request schema for project clone.""" + + git_url = fields.String(required=True) + depth = fields.Integer(missing=PROJECT_CLONE_DEPTH_DEFAULT) + + +class ProjectCloneContext(ProjectCloneRequest): + """Context schema for project clone.""" + + project_id = fields.String(missing=lambda: uuid.uuid4().hex) + name = fields.String(required=True) + fullname = fields.String(required=True) + email = fields.String(required=True) + owner = fields.String(required=True) + token = fields.String(required=True) + + @validates('git_url') + def validate_git_url(self, value): + """Validates git url.""" + try: + GitURL.parse(value) + except ConfigurationError as e: + raise ValidationError(str(e)) + + return value + + @post_load() + def format_url(self, data, **kwargs): + """Format URL with username and password.""" + git_url = urlparse(data['git_url']) + + url = 'oauth2:{0}@{1}'.format(data['token'], git_url.netloc) + data['url_with_auth'] = git_url._replace(netloc=url).geturl() + + return data + + @pre_load() + def set_owner_name(self, data, **kwargs): + """Set owner and name fields.""" + git_url = GitURL.parse(data['git_url']) + + data['owner'] = git_url.owner + data['name'] = git_url.name + + return data + + +class ProjectCloneResponse(Schema): + """Response schema for project clone.""" + + project_id = fields.String(required=True) + git_url = fields.String(required=True) + + +class ProjectCloneResponseRPC(JsonRPCResponse): + """RPC response schema for project clone response.""" + + result = fields.Nested(ProjectCloneResponse) + + +class ProjectListResponse(Schema): + """Response schema for project listing.""" + + projects = fields.List(fields.Nested(ProjectCloneResponse), required=True) + + +class ProjectListResponseRPC(JsonRPCResponse): + """RPC response schema for project listing.""" + + result = fields.Nested(ProjectListResponse) diff --git a/renku/service/serializers/datasets.py b/renku/service/serializers/datasets.py new file mode 100644 index 0000000000..154a0a8623 --- /dev/null +++ b/renku/service/serializers/datasets.py @@ -0,0 +1,131 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service datasets serializers.""" +from marshmallow import Schema, fields + +from renku.service.serializers.rpc import JsonRPCResponse + + +class DatasetAuthors(Schema): + """Schema for the dataset authors.""" + + name = fields.String(required=True) + affiliation = fields.String() + + +class DatasetCreateRequest(Schema): + """Request schema for dataset create view.""" + + dataset_name = fields.String(required=True) + description = fields.String() + authors = fields.List(fields.Nested(DatasetAuthors)) + project_id = fields.String(required=True) + + +class DatasetCreateResponse(Schema): + """Response schema for dataset create view.""" + + dataset_name = fields.String(required=True) + + +class DatasetCreateResponseRPC(JsonRPCResponse): + """RPC response schema for dataset create view.""" + + result = fields.Nested(DatasetCreateResponse) + + +class DatasetAddFile(Schema): + """Schema for dataset add file view.""" + + file_id = fields.String(required=True) + + +class DatasetAddRequest(Schema): + """Request schema for dataset add file view.""" + + dataset_name = fields.String(required=True) + create_dataset = fields.Boolean(missing=False) + project_id = fields.String(required=True) + files = fields.List(fields.Nested(DatasetAddFile), required=True) + + +class DatasetAddResponse(Schema): + """Response schema for dataset add file view.""" + + dataset_name = fields.String(required=True) + project_id = fields.String(required=True) + files = fields.List(fields.Nested(DatasetAddFile), required=True) + + +class DatasetAddResponseRPC(JsonRPCResponse): + """RPC schema for dataset add.""" + + result = fields.Nested(DatasetAddResponse) + + +class DatasetListRequest(Schema): + """Request schema for dataset list view.""" + + project_id = fields.String(required=True) + + +class DatasetDetails(Schema): + """Serialize dataset to response object.""" + + identifier = fields.String(required=True) + name = fields.String(required=True) + version = fields.String(allow_none=True) + created = fields.String(allow_none=True) + + +class DatasetListResponse(Schema): + """Response schema for dataset list view.""" + + datasets = fields.List(fields.Nested(DatasetDetails), required=True) + + +class DatasetListResponseRPC(JsonRPCResponse): + """RPC response schema for dataset list view.""" + + result = fields.Nested(DatasetListResponse) + + +class DatasetFilesListRequest(Schema): + """Request schema for dataset files list view.""" + + project_id = fields.String(required=True) + dataset_name = fields.String(required=True) + + +class DatasetFileDetails(Schema): + """Serialzie dataset files to response object.""" + + name = fields.String(required=True) + + +class DatasetFilesListResponse(Schema): + """Response schema for dataset files list view.""" + + dataset_name = fields.String(required=True) + files = fields.List(fields.Nested(DatasetFileDetails), required=True) + + +class DatasetFilesListResponseRPC(JsonRPCResponse): + """RPC schema for dataset files list view.""" + + result = fields.Nested(DatasetFilesListResponse) diff --git a/renku/service/serializers/headers.py b/renku/service/serializers/headers.py new file mode 100644 index 0000000000..f1ab332fb0 --- /dev/null +++ b/renku/service/serializers/headers.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service headers serializers.""" +from marshmallow import Schema, ValidationError, fields, pre_load +from werkzeug.utils import secure_filename + + +class UserIdentityHeaders(Schema): + """User identity schema.""" + + uid = fields.String(required=True, data_key='renku-user-id') + fullname = fields.String(data_key='renku-user-fullname') + email = fields.String(data_key='renku-user-email') + token = fields.String(data_key='authorization') + + def extract_token(self, data): + """Extract token.""" + value = data.get('authorization', '') + components = value.split(' ') + + rfc_compliant = value.lower().startswith('bearer') + rfc_compliant &= len(components) == 2 + + if not rfc_compliant: + raise ValidationError('authorization value contains invalid value') + + return components[-1] + + @pre_load() + def set_fields(self, data, **kwargs): + """Set fields for serialization.""" + expected_keys = [field.data_key for field in self.fields.values()] + + data = { + key.lower(): value + for key, value in data.items() if key.lower() in expected_keys + } + + if {'renku-user-id', 'authorization'}.issubset(set(data.keys())): + data['renku-user-id'] = secure_filename(data['renku-user-id']) + data['authorization'] = self.extract_token(data) + + return data diff --git a/renku/service/serializers/rpc.py b/renku/service/serializers/rpc.py new file mode 100644 index 0000000000..6512d26001 --- /dev/null +++ b/renku/service/serializers/rpc.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service JSON-RPC serializers.""" +from marshmallow import Schema, fields + + +class JsonRPCResponse(Schema): + """JsonRPC response schema.""" + + error = fields.Dict() diff --git a/renku/service/utils/__init__.py b/renku/service/utils/__init__.py new file mode 100644 index 0000000000..c5d87b539b --- /dev/null +++ b/renku/service/utils/__init__.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service utility functions.""" +from git import Repo + +from renku.service.config import CACHE_PROJECTS_PATH, CACHE_UPLOADS_PATH + + +def make_project_path(user, project): + """Construct full path for cached project.""" + valid_user = user and 'uid' in user + valid_project = project and 'owner' in project and 'name' in project + + if valid_user and valid_project: + return ( + CACHE_PROJECTS_PATH / user['uid'] / project['owner'] / + project['name'] + ) + + +def make_file_path(user, cached_file): + """Construct full path for cache file.""" + valid_user = user and 'uid' in user + valid_file = cached_file and 'file_name' in cached_file + + if valid_user and valid_file: + return CACHE_UPLOADS_PATH / user['uid'] / cached_file['relative_path'] + + +def repo_sync(repo_path, remote_names=('origin', )): + """Sync the repo with the remotes.""" + repo = Repo(repo_path) + is_pushed = False + + for remote in repo.remotes: + if remote.name in remote_names: + remote.push() + is_pushed = True + + return is_pushed diff --git a/renku/service/views/__init__.py b/renku/service/views/__init__.py new file mode 100644 index 0000000000..fbe49ab1d7 --- /dev/null +++ b/renku/service/views/__init__.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service views.""" diff --git a/renku/service/views/cache.py b/renku/service/views/cache.py new file mode 100644 index 0000000000..983c42d534 --- /dev/null +++ b/renku/service/views/cache.py @@ -0,0 +1,244 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service cache views.""" +import os +import shutil +from pathlib import Path + +import patoolib +from flask import Blueprint, jsonify, request +from flask_apispec import marshal_with, use_kwargs +from marshmallow import EXCLUDE +from patoolib.util import PatoolError + +from renku.core.commands.clone import renku_clone +from renku.service.config import CACHE_UPLOADS_PATH, \ + INVALID_PARAMS_ERROR_CODE, SUPPORTED_ARCHIVES +from renku.service.serializers.cache import FileListResponse, \ + FileListResponseRPC, FileUploadContext, FileUploadRequest, \ + FileUploadResponse, FileUploadResponseRPC, ProjectCloneContext, \ + ProjectCloneRequest, ProjectCloneResponse, ProjectCloneResponseRPC, \ + ProjectListResponse, ProjectListResponseRPC, extract_file +from renku.service.utils import make_file_path, make_project_path +from renku.service.views.decorators import accepts_json, handle_base_except, \ + handle_git_except, handle_renku_except, handle_validation_except, \ + header_doc, requires_cache, requires_identity + +CACHE_BLUEPRINT_TAG = 'cache' +cache_blueprint = Blueprint('cache', __name__) + + +@marshal_with(FileListResponseRPC) +@header_doc(description='List uploaded files.', tags=(CACHE_BLUEPRINT_TAG, )) +@cache_blueprint.route( + '/cache/files-list', + methods=['GET'], + provide_automatic_options=False, +) +@handle_base_except +@handle_git_except +@handle_renku_except +@handle_validation_except +@requires_cache +@requires_identity +def list_uploaded_files_view(user, cache): + """List uploaded files ready to be added to projects.""" + files = [ + f for f in cache.get_files(user['uid']) + if make_file_path(user, f).exists() + ] + + response = FileListResponseRPC().load({ + 'result': FileListResponse().load({'files': files}) + }) + return jsonify(response) + + +@use_kwargs(FileUploadRequest) +@marshal_with(FileUploadResponseRPC) +@header_doc( + description='Upload file or archive of files.', + tags=(CACHE_BLUEPRINT_TAG, ), +) +@cache_blueprint.route( + '/cache/files-upload', + methods=['POST'], + provide_automatic_options=False, +) +@handle_base_except +@handle_git_except +@handle_renku_except +@handle_validation_except +@requires_cache +@requires_identity +def upload_file_view(user, cache): + """Upload file or archive of files.""" + file = extract_file(request) + + response_builder = { + 'file_name': file.filename, + 'content_type': file.content_type, + 'is_archive': file.content_type in SUPPORTED_ARCHIVES + } + response_builder.update(FileUploadRequest().load(request.args)) + + user_cache_dir = CACHE_UPLOADS_PATH / user['uid'] + user_cache_dir.mkdir(exist_ok=True) + + file_path = user_cache_dir / file.filename + if file_path.exists(): + if response_builder.get('override_existing', False): + file_path.unlink() + else: + return jsonify( + error={ + 'code': INVALID_PARAMS_ERROR_CODE, + 'reason': 'file exists', + } + ) + + file.save(str(file_path)) + + files = [] + if response_builder['unpack_archive'] and response_builder['is_archive']: + unpack_dir = '{0}.unpacked'.format(file_path.name) + temp_dir = file_path.parent / Path(unpack_dir) + if temp_dir.exists(): + shutil.rmtree(str(temp_dir)) + temp_dir.mkdir(exist_ok=True) + + try: + patoolib.extract_archive(str(file_path), outdir=str(temp_dir)) + except PatoolError: + return jsonify( + error={ + 'code': INVALID_PARAMS_ERROR_CODE, + 'reason': 'unable to unpack archive' + } + ) + + for file_ in temp_dir.glob('**/*'): + file_obj = { + 'file_name': file_.name, + 'file_size': os.stat(str(file_path)).st_size, + 'relative_path': + str(file_.relative_to(CACHE_UPLOADS_PATH / user['uid'])) + } + + files.append(FileUploadContext().load(file_obj, unknown=EXCLUDE)) + + else: + response_builder['file_size'] = os.stat(str(file_path)).st_size + response_builder['relative_path'] = str( + file_path.relative_to(CACHE_UPLOADS_PATH / user['uid']) + ) + + files.append( + FileUploadContext().load(response_builder, unknown=EXCLUDE) + ) + + response = FileUploadResponseRPC().load({ + 'result': FileUploadResponse().load({'files': files}) + }) + cache.set_files(user['uid'], files) + + return jsonify(response) + + +@use_kwargs(ProjectCloneRequest) +@marshal_with(ProjectCloneResponseRPC) +@header_doc( + 'Clone a remote project. If the project is cached already, ' + 'new clone operation will override the old cache state.', + tags=(CACHE_BLUEPRINT_TAG, ) +) +@cache_blueprint.route( + '/cache/project-clone', + methods=['POST'], + provide_automatic_options=False, +) +@handle_base_except +@handle_git_except +@handle_renku_except +@handle_validation_except +@requires_cache +@requires_identity +@accepts_json +def project_clone(user, cache): + """Clone a remote repository.""" + ctx = ProjectCloneContext().load( + (lambda a, b: a.update(b) or a)(request.json, user), + unknown=EXCLUDE, + ) + + local_path = make_project_path(user, ctx) + + if local_path.exists(): + shutil.rmtree(str(local_path)) + + for project in cache.get_projects(user['uid']): + if project['git_url'] == ctx['git_url']: + cache.invalidate_project(user['uid'], project['project_id']) + + local_path.mkdir(parents=True, exist_ok=True) + renku_clone( + ctx['url_with_auth'], + local_path, + depth=ctx['depth'], + raise_git_except=True, + config={ + 'user.name': ctx['fullname'], + 'user.email': ctx['email'], + } + ) + cache.set_project(user['uid'], ctx['project_id'], ctx) + + response = ProjectCloneResponseRPC().load({ + 'result': ProjectCloneResponse().load(ctx, unknown=EXCLUDE) + }) + return jsonify(response) + + +@marshal_with(ProjectListResponseRPC) +@header_doc( + 'List cached projects.', + tags=(CACHE_BLUEPRINT_TAG, ), +) +@cache_blueprint.route( + '/cache/project-list', + methods=['GET'], + provide_automatic_options=False, +) +@handle_base_except +@handle_git_except +@handle_renku_except +@handle_validation_except +@requires_cache +@requires_identity +def list_projects_view(user, cache): + """List cached projects.""" + projects = cache.get_projects(user['uid']) + projects = [ + ProjectCloneResponse().load(p, unknown=EXCLUDE) + for p in projects if make_project_path(user, p).exists() + ] + + response = ProjectListResponseRPC().load({ + 'result': ProjectListResponse().load({'projects': projects}) + }) + return jsonify(response) diff --git a/renku/service/views/datasets.py b/renku/service/views/datasets.py new file mode 100644 index 0000000000..3984ee8877 --- /dev/null +++ b/renku/service/views/datasets.py @@ -0,0 +1,235 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service datasets view.""" +import json + +from flask import Blueprint, jsonify, request +from flask_apispec import marshal_with, use_kwargs +from marshmallow import EXCLUDE + +from renku.core.commands.dataset import add_file, create_dataset, \ + dataset_parent, list_files +from renku.core.utils.contexts import chdir +from renku.service.config import INTERNAL_FAILURE_ERROR_CODE, \ + INVALID_PARAMS_ERROR_CODE +from renku.service.serializers.datasets import DatasetAddRequest, \ + DatasetAddResponse, DatasetAddResponseRPC, DatasetCreateRequest, \ + DatasetCreateResponse, DatasetCreateResponseRPC, DatasetDetails, \ + DatasetFileDetails, DatasetFilesListRequest, DatasetFilesListResponse, \ + DatasetFilesListResponseRPC, DatasetListRequest, DatasetListResponse, \ + DatasetListResponseRPC +from renku.service.utils import make_file_path, make_project_path, repo_sync +from renku.service.views.decorators import accepts_json, handle_base_except, \ + handle_git_except, handle_renku_except, handle_validation_except, \ + header_doc, requires_cache, requires_identity + +DATASET_BLUEPRINT_TAG = 'datasets' +dataset_blueprint = Blueprint(DATASET_BLUEPRINT_TAG, __name__) + + +@use_kwargs(DatasetListRequest, locations=['query']) +@marshal_with(DatasetListResponseRPC) +@header_doc('List all datasets in project.', tags=(DATASET_BLUEPRINT_TAG, )) +@dataset_blueprint.route( + '/datasets/list', + methods=['GET'], + provide_automatic_options=False, +) +@handle_base_except +@handle_git_except +@handle_renku_except +@handle_validation_except +@requires_cache +@requires_identity +def list_datasets_view(user, cache): + """List all datasets in project.""" + req = DatasetListRequest().load(request.args) + project = cache.get_project(user['uid'], req['project_id']) + project_path = make_project_path(user, project) + + if not project_path: + return jsonify( + error={ + 'code': INVALID_PARAMS_ERROR_CODE, + 'reason': 'invalid project_id argument', + } + ) + + with chdir(project_path): + datasets = [ + DatasetDetails().load(ds, unknown=EXCLUDE) + # TODO: fix core interface to address this issue (add ticket ref) + for ds in json.loads(dataset_parent(None, 'data', 'json-ld')) + ] + + response = DatasetListResponse().load({'datasets': datasets}) + return jsonify(DatasetListResponseRPC().load({'result': response})) + + +@use_kwargs(DatasetFilesListRequest, locations=['query']) +@marshal_with(DatasetFilesListResponseRPC) +@header_doc('List files in a dataset.', tags=(DATASET_BLUEPRINT_TAG, )) +@dataset_blueprint.route( + '/datasets/files-list', + methods=['GET'], + provide_automatic_options=False, +) +@handle_base_except +@handle_git_except +@handle_renku_except +@handle_validation_except +@requires_cache +@requires_identity +def list_dataset_files_view(user, cache): + """List files in a dataset.""" + ctx = DatasetFilesListRequest().load(request.args) + project = cache.get_project(user['uid'], ctx['project_id']) + project_path = make_project_path(user, project) + + if not project_path: + return jsonify( + error={ + 'code': INVALID_PARAMS_ERROR_CODE, + 'reason': 'invalid project_id argument', + } + ) + + with chdir(project_path): + dataset_files = json.loads( + # TODO: fix core interface to address this issue (add ticket ref) + list_files(ctx['dataset_name'], None, None, None, 'json-ld') + ) + ctx['files'] = [ + DatasetFileDetails().load(ds, unknown=EXCLUDE) + for ds in dataset_files + ] + + response = DatasetFilesListResponse().load(ctx, unknown=EXCLUDE) + return jsonify(DatasetFilesListResponseRPC().load({'result': response})) + + +@use_kwargs(DatasetAddRequest) +@marshal_with(DatasetAddResponseRPC) +@header_doc( + 'Add uploaded file to cloned repository.', tags=(DATASET_BLUEPRINT_TAG, ) +) +@dataset_blueprint.route( + '/datasets/add', + methods=['POST'], + provide_automatic_options=False, +) +@handle_base_except +@handle_git_except +@handle_renku_except +@handle_validation_except +@accepts_json +@requires_cache +@requires_identity +def add_file_to_dataset_view(user, cache): + """Add uploaded file to cloned repository.""" + ctx = DatasetAddRequest().load(request.json) + project = cache.get_project(user['uid'], ctx['project_id']) + project_path = make_project_path(user, project) + if not project_path: + return jsonify( + error={ + 'code': INVALID_PARAMS_ERROR_CODE, + 'message': 'invalid project_id: {0}'.format(ctx['project_id']), + } + ) + + local_paths = [] + for file_ in ctx['files']: + file = cache.get_file(user['uid'], file_['file_id']) + local_path = make_file_path(user, file) + if not local_path or not local_path.exists(): + return jsonify( + error={ + 'code': INVALID_PARAMS_ERROR_CODE, + 'message': 'invalid file_id: {0}'.format(file_['file_id']) + } + ) + local_paths.append(str(local_path)) + + with chdir(project_path): + add_file( + local_paths, ctx['dataset_name'], create=ctx['create_dataset'] + ) + + if not repo_sync(project_path): + return jsonify( + error={ + 'code': INTERNAL_FAILURE_ERROR_CODE, + 'message': 'repo sync failed' + } + ) + + return jsonify( + DatasetAddResponseRPC().load({ + 'result': DatasetAddResponse().load(ctx, unknown=EXCLUDE) + }) + ) + + +@use_kwargs(DatasetCreateRequest) +@marshal_with(DatasetCreateResponseRPC) +@header_doc( + 'Create a new dataset in a project.', tags=(DATASET_BLUEPRINT_TAG, ) +) +@dataset_blueprint.route( + '/datasets/create', + methods=['POST'], + provide_automatic_options=False, +) +@handle_base_except +@handle_git_except +@handle_renku_except +@handle_validation_except +@accepts_json +@requires_cache +@requires_identity +def create_dataset_view(user, cache): + """Create a new dataset in a project.""" + ctx = DatasetCreateRequest().load(request.json) + project = cache.get_project(user['uid'], ctx['project_id']) + + project_path = make_project_path(user, project) + if not project_path: + return jsonify( + error={ + 'code': INVALID_PARAMS_ERROR_CODE, + 'message': 'invalid project_id argument', + } + ) + + with chdir(project_path): + create_dataset(ctx['dataset_name']) + + if not repo_sync(project_path): + return jsonify( + error={ + 'code': INTERNAL_FAILURE_ERROR_CODE, + 'reason': 'push to remote failed silently - try again' + } + ) + + return jsonify( + DatasetCreateResponseRPC().load({ + 'result': DatasetCreateResponse().load(ctx, unknown=EXCLUDE) + }) + ) diff --git a/renku/service/views/decorators.py b/renku/service/views/decorators.py new file mode 100644 index 0000000000..517574bedc --- /dev/null +++ b/renku/service/views/decorators.py @@ -0,0 +1,250 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service view decorators.""" +from functools import wraps + +from flask import current_app, jsonify, request +from flask_apispec import doc +from git import GitCommandError +from marshmallow import ValidationError +from redis import RedisError + +from renku.core.errors import RenkuException +from renku.service.config import GIT_ACCESS_DENIED_ERROR_CODE, \ + GIT_UNKNOWN_ERROR_CODE, INTERNAL_FAILURE_ERROR_CODE, \ + INVALID_HEADERS_ERROR_CODE, INVALID_PARAMS_ERROR_CODE, \ + REDIS_EXCEPTION_ERROR_CODE, RENKU_EXCEPTION_ERROR_CODE +from renku.service.serializers.headers import UserIdentityHeaders + + +def requires_identity(f): + """Wrapper which indicates that route requires user identification.""" + # noqa + @wraps(f) + def decorated_function(*args, **kws): + """Represents decorated function.""" + try: + user = UserIdentityHeaders().load(request.headers) + except (ValidationError, KeyError): + err_message = 'user identification is incorrect or missing' + return jsonify( + error={ + 'code': INVALID_HEADERS_ERROR_CODE, + 'reason': err_message + } + ) + + return f(user, *args, **kws) + + return decorated_function + + +def handle_redis_except(f): + """Wrapper which handles Redis exceptions.""" + # noqa + @wraps(f) + def decorated_function(*args, **kwargs): + """Represents decorated function.""" + try: + return f(*args, **kwargs) + except (RedisError, OSError) as e: + error_code = REDIS_EXCEPTION_ERROR_CODE + + return jsonify(error={ + 'code': error_code, + 'reason': e.messages, + }) + + return decorated_function + + +@handle_redis_except +def requires_cache(f): + """Wrapper which injects cache object into view.""" + # noqa + @wraps(f) + def decorated_function(*args, **kwargs): + """Represents decorated function.""" + return f(current_app.config.get('cache'), *args, **kwargs) + + return decorated_function + + +def handle_validation_except(f): + """Wrapper which handles marshmallow `ValidationError`.""" + # noqa + @wraps(f) + def decorated_function(*args, **kwargs): + """Represents decorated function.""" + try: + return f(*args, **kwargs) + except ValidationError as e: + return jsonify( + error={ + 'code': INVALID_PARAMS_ERROR_CODE, + 'reason': e.messages, + } + ) + + return decorated_function + + +def handle_renku_except(f): + """Wrapper which handles `RenkuException`.""" + # noqa + @wraps(f) + def decorated_function(*args, **kwargs): + """Represents decorated function.""" + try: + return f(*args, **kwargs) + except RenkuException as e: + return jsonify( + error={ + 'code': RENKU_EXCEPTION_ERROR_CODE, + 'reason': str(e), + } + ) + + return decorated_function + + +def handle_git_except(f): + """Wrapper which handles `RenkuException`.""" + # noqa + @wraps(f) + def decorated_function(*args, **kwargs): + """Represents decorated function.""" + try: + return f(*args, **kwargs) + except GitCommandError as e: + + error_code = GIT_ACCESS_DENIED_ERROR_CODE \ + if 'Access denied' in e.stderr else GIT_UNKNOWN_ERROR_CODE + + return jsonify( + error={ + 'code': error_code, + 'reason': + 'git error: {0}'. + format(' '.join(e.stderr.strip().split('\n'))), + } + ) + + return decorated_function + + +def accepts_json(f): + """Wrapper which ensures only JSON payload can be in request.""" + # noqa + @wraps(f) + def decorated_function(*args, **kwargs): + """Represents decorated function.""" + if 'Content-Type' not in request.headers: + return jsonify( + error={ + 'code': INVALID_HEADERS_ERROR_CODE, + 'reason': 'invalid request headers' + } + ) + + header_check = request.headers['Content-Type'] == 'application/json' + + if not request.is_json or not header_check: + return jsonify( + error={ + 'code': INVALID_HEADERS_ERROR_CODE, + 'reason': 'invalid request payload' + } + ) + + return f(*args, **kwargs) + + return decorated_function + + +def handle_base_except(f): + """Wrapper which handles base exceptions.""" + # noqa + @wraps(f) + def decorated_function(*args, **kwargs): + """Represents decorated function.""" + try: + return f(*args, **kwargs) + except (Exception, BaseException, OSError) as e: + error_code = INTERNAL_FAILURE_ERROR_CODE + + return jsonify( + error={ + 'code': error_code, + 'reason': + 'internal error: {0}'. + format(' '.join(e.stderr.strip().split('\n'))), + } + ) + + return decorated_function + + +def header_doc(description, tags=()): + """Wrap additional OpenAPI header description for an endpoint.""" + return doc( + description=description, + params={ + 'Authorization': { + 'description': ( + 'Used for users git oauth2 access. ' + 'For example: ' + '```Authorization: Bearer asdf-qwer-zxcv```' + ), + 'in': 'header', + 'type': 'string', + 'required': True + }, + 'Renku-User-Id': { + 'description': ( + 'Used for identification of the users. ' + 'For example: ' + '```Renku-User-Id: sasdsa-sadsd-gsdsdh-gfdgdsd```' + ), + 'in': 'header', + 'type': 'string', + 'required': True + }, + 'Renku-User-FullName': { + 'description': ( + 'Used for commit author signature. ' + 'For example: ' + '```Renku-User-FullName: Rok Roskar```' + ), + 'in': 'header', + 'type': 'string', + 'required': True + }, + 'Renku-User-Email': { + 'description': ( + 'Used for commit author signature. ' + 'For example: ' + '```Renku-User-Email: dev@renkulab.io```' + ), + 'in': 'header', + 'type': 'string', + 'required': True + }, + }, + tags=list(tags), + ) diff --git a/setup.py b/setup.py index ae0d86b42e..8e799749d4 100644 --- a/setup.py +++ b/setup.py @@ -27,9 +27,11 @@ tests_require = [ 'check-manifest>=0.37', 'coverage>=4.5.3', + 'fakeredis==1.1.0', 'flake8>=3.5', 'freezegun>=0.3.12', 'isort==4.3.4', + 'six>=1.13.0', 'pydocstyle>=3.0.0', 'pytest-cache>=1.0', 'pytest-cov>=2.5.1', @@ -73,6 +75,7 @@ install_requires = [ 'appdirs>=1.4.3', + 'apispec==3.0.0', 'attrs>=18.2.0', 'click-completion>=0.5.0', 'click>=7.0', @@ -80,6 +83,9 @@ 'cwltool==1.0.20181012180214', 'environ_config>=18.2.0', 'filelock>=3.0.0', + 'flask==1.1.1', + 'flask-apispec==0.8.3', + 'flask-swagger-ui==3.20.9', 'gitpython==3.0.3', 'patool>=1.12', 'psutil>=5.4.7', @@ -90,9 +96,11 @@ 'pyshacl>=0.11.3.post1', 'python-dateutil>=2.6.1', 'python-editor>=1.0.4', + 'redis==3.3.11', 'rdflib-jsonld>=0.4.0', 'requests>=2.21.0', 'ndg-httpsclient>=0.5.1', + 'marshmallow==3.2.2', 'idna>=2.8', 'setuptools_scm>=3.1.0', 'tabulate>=0.7.7', diff --git a/tests/cli/test_datasets.py b/tests/cli/test_datasets.py index a7efdb7890..23831ad075 100644 --- a/tests/cli/test_datasets.py +++ b/tests/cli/test_datasets.py @@ -1196,8 +1196,8 @@ def test_avoid_empty_commits(runner, client, directory_tree): def test_add_removes_credentials(runner, client): """Test credentials are removed when adding to a dataset.""" - URL = 'https://username:password@example.com/index.html' - result = runner.invoke(cli, ['dataset', 'add', '-c', 'my-dataset', URL]) + url = 'https://username:password@example.com/index.html' + result = runner.invoke(cli, ['dataset', 'add', '-c', 'my-dataset', url]) assert 0 == result.exit_code with client.with_dataset('my-dataset') as dataset: diff --git a/tests/service/test_cache_views.py b/tests/service/test_cache_views.py new file mode 100644 index 0000000000..dcc066cf64 --- /dev/null +++ b/tests/service/test_cache_views.py @@ -0,0 +1,599 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service cache view tests.""" +import io +import json +import uuid + +import pytest + +from renku.core.models.git import GitURL +from renku.service.config import INVALID_HEADERS_ERROR_CODE, \ + INVALID_PARAMS_ERROR_CODE + +REMOTE_URL = 'https://dev.renku.ch/gitlab/contact/integration-tests' +PERSONAL_ACCESS_TOKEN = 'LkoLiyLqnhMCAa4or5qa' + + +@pytest.mark.service +def test_serve_api_spec(svc_client): + """Check serving of service spec.""" + headers = { + 'Content-Type': 'application/json', + 'accept': 'application/json', + } + response = svc_client.get('/api/v1/spec', headers=headers) + + assert 0 != len(response.json.keys()) + assert 200 == response.status_code + + +@pytest.mark.service +def test_list_upload_files_all(svc_client): + """Check list uploaded files view.""" + headers_user = { + 'Content-Type': 'application/json', + 'accept': 'application/json', + 'Renku-User-Id': 'user' + } + response = svc_client.get('/cache/files-list', headers=headers_user) + + assert {'result'} == set(response.json.keys()) + + assert 0 == len(response.json['result']['files']) + assert 200 == response.status_code + + +@pytest.mark.service +def test_list_upload_files_all_no_auth(svc_client): + """Check error response on list uploaded files view.""" + headers = { + 'Content-Type': 'application/json', + 'accept': 'application/json', + } + response = svc_client.get( + '/cache/files-list', + headers=headers, + ) + + assert 200 == response.status_code + + assert {'error'} == set(response.json.keys()) + assert INVALID_HEADERS_ERROR_CODE == response.json['error']['code'] + + +@pytest.mark.service +def test_file_upload(svc_client): + """Check successful file upload.""" + headers_user = {'Renku-User-Id': '{0}'.format(uuid.uuid4().hex)} + + response = svc_client.post( + '/cache/files-upload', + data=dict(file=(io.BytesIO(b'this is a test'), 'datafile.txt'), ), + headers=headers_user, + ) + + assert response + assert 200 == response.status_code + + assert {'result'} == set(response.json.keys()) + assert isinstance( + uuid.UUID(response.json['result']['files'][0]['file_id']), uuid.UUID + ) + + +@pytest.mark.service +def test_file_upload_override(svc_client): + """Check successful file upload.""" + headers_user = {'Renku-User-Id': '{0}'.format(uuid.uuid4().hex)} + + response = svc_client.post( + '/cache/files-upload', + data=dict(file=(io.BytesIO(b'this is a test'), 'datafile.txt'), ), + headers=headers_user, + ) + + assert response + assert 200 == response.status_code + + assert {'result'} == set(response.json.keys()) + assert isinstance( + uuid.UUID(response.json['result']['files'][0]['file_id']), uuid.UUID + ) + old_file_id = response.json['result']['files'][0]['file_id'] + + response = svc_client.post( + '/cache/files-upload', + data=dict(file=(io.BytesIO(b'this is a test'), 'datafile.txt'), ), + headers=headers_user, + ) + + assert response + assert 200 == response.status_code + + assert {'error'} == set(response.json.keys()) + assert INVALID_PARAMS_ERROR_CODE == response.json['error']['code'] + assert 'file exists' == response.json['error']['reason'] + + response = svc_client.post( + '/cache/files-upload', + data=dict(file=(io.BytesIO(b'this is a test'), 'datafile.txt'), ), + query_string={'override_existing': True}, + headers=headers_user, + ) + + assert response + assert 200 == response.status_code + + assert {'result'} == set(response.json.keys()) + assert isinstance( + uuid.UUID(response.json['result']['files'][0]['file_id']), uuid.UUID + ) + assert old_file_id != response.json['result']['files'][0]['file_id'] + + +@pytest.mark.service +def test_file_upload_same_file(svc_client): + """Check successful file upload with same file uploaded twice.""" + headers_user1 = {'Renku-User-Id': '{0}'.format(uuid.uuid4().hex)} + response = svc_client.post( + '/cache/files-upload', + data=dict(file=(io.BytesIO(b'this is a test'), 'datafile.txt'), ), + headers=headers_user1, + ) + + assert response + assert 200 == response.status_code + + assert {'result'} == set(response.json.keys()) + + assert isinstance( + uuid.UUID(response.json['result']['files'][0]['file_id']), uuid.UUID + ) + + response = svc_client.post( + '/cache/files-upload', + data=dict(file=(io.BytesIO(b'this is a test'), 'datafile.txt'), ), + headers=headers_user1, + ) + + assert response + assert 200 == response.status_code + assert {'error'} == set(response.json.keys()) + assert INVALID_PARAMS_ERROR_CODE == response.json['error']['code'] + assert 'file exists' == response.json['error']['reason'] + + +@pytest.mark.service +def test_file_upload_no_auth(svc_client): + """Check failed file upload.""" + response = svc_client.post( + '/cache/files-upload', + data=dict(file=(io.BytesIO(b'this is a test'), 'datafile.txt'), ), + ) + + assert response + assert 200 == response.status_code + + assert {'error'} == set(response.json.keys()) + assert INVALID_HEADERS_ERROR_CODE == response.json['error']['code'] + + +@pytest.mark.service +def test_file_upload_with_users(svc_client): + """Check successful file upload and listing based on user auth header.""" + headers_user1 = {'Renku-User-Id': '{0}'.format(uuid.uuid4().hex)} + headers_user2 = {'Renku-User-Id': '{0}'.format(uuid.uuid4().hex)} + + response = svc_client.post( + '/cache/files-upload', + data=dict(file=(io.BytesIO(b'this is a test'), 'datafile1.txt'), ), + headers=headers_user1 + ) + + assert {'result'} == set(response.json.keys()) + + file_id = response.json['result']['files'][0]['file_id'] + assert file_id + assert 200 == response.status_code + + response = svc_client.post( + '/cache/files-upload', + data=dict(file=(io.BytesIO(b'this is a test'), 'datafile1.txt'), ), + headers=headers_user2 + ) + + assert response + assert {'result'} == set(response.json.keys()) + + response = svc_client.get('/cache/files-list', headers=headers_user1) + + assert response + + assert {'result'} == set(response.json.keys()) + assert 1 == len(response.json['result']['files']) + + file = response.json['result']['files'][0] + assert file_id == file['file_id'] + assert 0 < file['file_size'] + + +@pytest.mark.service +@pytest.mark.integration +def test_clone_projects_no_auth(svc_client): + """Check error on cloning of remote repository.""" + payload = { + 'git_url': REMOTE_URL, + } + + response = svc_client.post( + '/cache/project-clone', data=json.dumps(payload) + ) + + assert {'error'} == set(response.json.keys()) + assert INVALID_HEADERS_ERROR_CODE == response.json['error']['code'] + + err_message = 'user identification is incorrect or missing' + assert err_message == response.json['error']['reason'] + + headers = { + 'Content-Type': 'application/json', + 'accept': 'application/json', + 'Renku-User-Id': '{0}'.format(uuid.uuid4().hex), + 'Renku-User-FullName': 'Just Sam', + 'Renku-User-Email': 'contact@justsam.io', + 'Authorization': 'Bearer notatoken', + } + + response = svc_client.post( + '/cache/project-clone', data=json.dumps(payload), headers=headers + ) + + assert response + assert {'result'} == set(response.json.keys()) + + +@pytest.mark.service +@pytest.mark.integration +def test_clone_projects_with_auth(svc_client): + """Check cloning of remote repository.""" + headers = { + 'Content-Type': 'application/json', + 'accept': 'application/json', + 'Renku-User-Id': '{0}'.format(uuid.uuid4().hex), + 'Renku-User-FullName': 'Just Sam', + 'Renku-User-Email': 'contact@justsam.io', + 'Authorization': 'Bearer {0}'.format(PERSONAL_ACCESS_TOKEN), + } + + payload = { + 'git_url': REMOTE_URL, + } + + response = svc_client.post( + '/cache/project-clone', data=json.dumps(payload), headers=headers + ) + + assert response + assert {'result'} == set(response.json.keys()) + + +@pytest.mark.service +@pytest.mark.integration +def test_clone_projects_multiple(svc_client): + """Check multiple cloning of remote repository.""" + project_ids = [] + + headers = { + 'Content-Type': 'application/json', + 'Renku-User-Id': '{0}'.format(uuid.uuid4().hex), + 'Renku-User-FullName': 'Just Sam', + 'Renku-User-Email': 'contact@justsam.io', + 'Authorization': 'Bearer {0}'.format(PERSONAL_ACCESS_TOKEN), + } + + payload = { + 'git_url': REMOTE_URL, + } + + response = svc_client.post( + '/cache/project-clone', data=json.dumps(payload), headers=headers + ) + + assert response + assert {'result'} == set(response.json.keys()) + project_ids.append(response.json['result']) + + response = svc_client.post( + '/cache/project-clone', data=json.dumps(payload), headers=headers + ) + + assert response + assert {'result'} == set(response.json.keys()) + project_ids.append(response.json['result']) + + response = svc_client.post( + '/cache/project-clone', data=json.dumps(payload), headers=headers + ) + + assert response + assert {'result'} == set(response.json.keys()) + project_ids.append(response.json['result']) + + response = svc_client.post( + '/cache/project-clone', data=json.dumps(payload), headers=headers + ) + + assert response + assert {'result'} == set(response.json.keys()) + last_pid = response.json['result']['project_id'] + + response = svc_client.get('/cache/project-list', headers=headers) + + assert response + assert {'result'} == set(response.json.keys()) + + pids = [p['project_id'] for p in response.json['result']['projects']] + assert last_pid in pids + + for inserted in project_ids: + assert inserted['project_id'] not in pids + + +@pytest.mark.service +@pytest.mark.integration +def test_clone_projects_list_view_errors(svc_client): + """Check cache state of cloned projects with no headers.""" + headers = { + 'Content-Type': 'application/json', + 'Renku-User-Id': '{0}'.format(uuid.uuid4().hex), + 'Renku-User-FullName': 'Just Sam', + 'Renku-User-Email': 'contact@justsam.io', + 'Authorization': 'Bearer {0}'.format(PERSONAL_ACCESS_TOKEN), + } + + payload = { + 'git_url': REMOTE_URL, + } + + response = svc_client.post( + '/cache/project-clone', data=json.dumps(payload), headers=headers + ) + assert response + assert {'result'} == set(response.json.keys()) + + assert isinstance( + uuid.UUID(response.json['result']['project_id']), uuid.UUID + ) + + response = svc_client.get( + '/cache/project-list', + # no auth headers, expected error + ) + assert response + + assert {'error'} == set(response.json.keys()) + assert INVALID_HEADERS_ERROR_CODE == response.json['error']['code'] + + response = svc_client.get('/cache/project-list', headers=headers) + assert response + assert {'result'} == set(response.json.keys()) + assert 1 == len(response.json['result']['projects']) + + project = response.json['result']['projects'][0] + assert isinstance(uuid.UUID(project['project_id']), uuid.UUID) + assert isinstance(GitURL.parse(project['git_url']), GitURL) + + +@pytest.mark.service +@pytest.mark.integration +def test_clone_projects_invalid_headers(svc_client): + """Check cache state of cloned projects with invalid headers.""" + headers = { + 'Content-Type': 'application/json', + 'accept': 'application/json', + 'Renku-User-Id': '{0}'.format(uuid.uuid4().hex), + 'Renku-User-FullName': 'Not Sam', + 'Renku-User-Email': 'not@sam.io', + 'Authorization': 'Bearer not-a-token', + } + + payload = { + 'git_url': REMOTE_URL, + } + + response = svc_client.post( + '/cache/project-clone', + data=json.dumps(payload), + headers=headers, + ) + assert response + assert {'result'} == set(response.json.keys()) + + response = svc_client.get( + '/cache/project-list', + # no auth headers, expected error + ) + + assert response + assert {'error'} == set(response.json.keys()) + assert INVALID_HEADERS_ERROR_CODE == response.json['error']['code'] + + response = svc_client.get('/cache/project-list', headers=headers) + assert response + assert {'result'} == set(response.json.keys()) + assert 1 == len(response.json['result']['projects']) + + +@pytest.mark.service +def test_upload_zip_unpack_archive(datapack_zip, svc_client_with_repo): + """Upload zip archive with unpack.""" + svc_client, headers, project_id = svc_client_with_repo + headers.pop('Content-Type') + + response = svc_client.post( + '/cache/files-upload', + data=dict( + file=(io.BytesIO(datapack_zip.read_bytes()), datapack_zip.name), + ), + query_string={ + 'unpack_archive': True, + 'override_existing': True, + }, + headers=headers + ) + + assert response + + assert 200 == response.status_code + assert {'result'} == set(response.json.keys()) + assert 2 == len(response.json['result']['files']) + + for file_ in response.json['result']['files']: + assert not file_['is_archive'] + assert not file_['unpack_archive'] + + +@pytest.mark.service +def test_upload_zip_archive(datapack_zip, svc_client_with_repo): + """Upload zip archive.""" + svc_client, headers, project_id = svc_client_with_repo + headers.pop('Content-Type') + + response = svc_client.post( + '/cache/files-upload', + data=dict( + file=(io.BytesIO(datapack_zip.read_bytes()), datapack_zip.name), + ), + query_string={ + 'unpack_archive': False, + 'override_existing': True, + }, + headers=headers + ) + + assert response + + assert 200 == response.status_code + assert {'result'} == set(response.json.keys()) + assert 1 == len(response.json['result']['files']) + + for file_ in response.json['result']['files']: + assert file_['is_archive'] + assert not file_['unpack_archive'] + + +@pytest.mark.service +def test_upload_tar_unpack_archive(datapack_tar, svc_client_with_repo): + """Upload zip archive with unpack.""" + svc_client, headers, project_id = svc_client_with_repo + headers.pop('Content-Type') + + response = svc_client.post( + '/cache/files-upload', + data=dict( + file=(io.BytesIO(datapack_tar.read_bytes()), datapack_tar.name), + ), + query_string={ + 'unpack_archive': True, + 'override_existing': True, + }, + headers=headers + ) + + assert response + + assert 200 == response.status_code + assert {'result'} == set(response.json.keys()) + assert 2 == len(response.json['result']['files']) + + for file_ in response.json['result']['files']: + assert not file_['is_archive'] + assert not file_['unpack_archive'] + + +@pytest.mark.service +def test_upload_tar_archive(datapack_tar, svc_client_with_repo): + """Upload zip archive.""" + svc_client, headers, project_id = svc_client_with_repo + headers.pop('Content-Type') + + response = svc_client.post( + '/cache/files-upload', + data=dict( + file=(io.BytesIO(datapack_tar.read_bytes()), datapack_tar.name), + ), + query_string={ + 'unpack_archive': False, + 'override_existing': True, + }, + headers=headers + ) + + assert response + + assert 200 == response.status_code + assert {'result'} == set(response.json.keys()) + assert 1 == len(response.json['result']['files']) + + for file_ in response.json['result']['files']: + assert file_['is_archive'] + assert not file_['unpack_archive'] + + +@pytest.mark.service +def test_field_upload_resp_fields(datapack_tar, svc_client_with_repo): + """Check response fields.""" + svc_client, headers, project_id = svc_client_with_repo + headers.pop('Content-Type') + + response = svc_client.post( + '/cache/files-upload', + data=dict( + file=(io.BytesIO(datapack_tar.read_bytes()), datapack_tar.name), + ), + query_string={ + 'unpack_archive': True, + 'override_existing': True, + }, + headers=headers + ) + + assert response + + assert 200 == response.status_code + + assert {'result'} == set(response.json.keys()) + assert 2 == len(response.json['result']['files']) + assert { + 'content_type', + 'file_id', + 'file_name', + 'file_size', + 'is_archive', + 'timestamp', + 'is_archive', + 'unpack_archive', + 'relative_path', + } == set(response.json['result']['files'][0].keys()) + + assert not response.json['result']['files'][0]['is_archive'] + assert not response.json['result']['files'][0]['unpack_archive'] + + rel_path = response.json['result']['files'][0]['relative_path'] + assert rel_path.startswith(datapack_tar.name) and 'unpacked' in rel_path diff --git a/tests/service/test_dataset_views.py b/tests/service/test_dataset_views.py new file mode 100644 index 0000000000..60780daa6b --- /dev/null +++ b/tests/service/test_dataset_views.py @@ -0,0 +1,547 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service dataset view tests.""" +import io +import json +import uuid + +import pytest + +from renku.service.config import INVALID_HEADERS_ERROR_CODE, \ + INVALID_PARAMS_ERROR_CODE, RENKU_EXCEPTION_ERROR_CODE + + +@pytest.mark.service +@pytest.mark.integration +def test_create_dataset_view(svc_client_with_repo): + """Create new dataset successfully.""" + svc_client, headers, project_id = svc_client_with_repo + + payload = { + 'project_id': project_id, + 'dataset_name': '{0}'.format(uuid.uuid4().hex), + } + + response = svc_client.post( + '/datasets/create', + data=json.dumps(payload), + headers=headers, + ) + + assert response + + assert {'result'} == set(response.json.keys()) + assert {'dataset_name'} == set(response.json['result'].keys()) + assert payload['dataset_name'] == response.json['result']['dataset_name'] + + +@pytest.mark.service +@pytest.mark.integration +def test_create_dataset_view_dataset_exists(svc_client_with_repo): + """Create new dataset which already exists.""" + svc_client, headers, project_id = svc_client_with_repo + + payload = { + 'project_id': project_id, + 'dataset_name': 'mydataset', + } + + response = svc_client.post( + '/datasets/create', + data=json.dumps(payload), + headers=headers, + ) + + assert response + assert {'error'} == set(response.json.keys()) + + assert RENKU_EXCEPTION_ERROR_CODE == response.json['error']['code'] + assert 'Dataset exists' in response.json['error']['reason'] + + +@pytest.mark.service +@pytest.mark.integration +def test_create_dataset_view_unknown_param(svc_client_with_repo): + """Create new dataset by specifying unknown parameters.""" + svc_client, headers, project_id = svc_client_with_repo + + payload = { + 'project_id': project_id, + 'dataset_name': 'mydata', + 'remote_name': 'origin' + } + + response = svc_client.post( + '/datasets/create', + data=json.dumps(payload), + headers=headers, + ) + + assert response + assert {'error'} == set(response.json.keys()) + + assert INVALID_PARAMS_ERROR_CODE == response.json['error']['code'] + assert {'remote_name'} == set(response.json['error']['reason'].keys()) + + +@pytest.mark.service +@pytest.mark.integration +def test_create_dataset_with_no_identity(svc_client_with_repo): + """Create new dataset with no identification provided.""" + svc_client, headers, project_id = svc_client_with_repo + + payload = { + 'project_id': project_id, + 'dataset_name': 'mydata', + 'remote_name': 'origin', + } + + response = svc_client.post( + '/datasets/create', + data=json.dumps(payload), + headers={'Content-Type': headers['Content-Type']} + # no user identity, expect error + ) + + assert response + assert {'error'} == response.json.keys() + + assert INVALID_HEADERS_ERROR_CODE == response.json['error']['code'] + + err_message = 'user identification is incorrect or missing' + assert err_message == response.json['error']['reason'] + + +@pytest.mark.service +@pytest.mark.integration +def test_add_file_view_with_no_identity(svc_client_with_repo): + """Check identity error raise in dataset add.""" + svc_client, headers, project_id = svc_client_with_repo + payload = { + 'project_id': project_id, + 'dataset_name': 'mydata', + 'remote_name': 'origin', + } + + response = svc_client.post( + '/datasets/add', + data=json.dumps(payload), + headers={'Content-Type': headers['Content-Type']} + # no user identity, expect error + ) + assert response + + assert {'error'} == set(response.json.keys()) + assert INVALID_HEADERS_ERROR_CODE == response.json['error']['code'] + + err_message = 'user identification is incorrect or missing' + assert err_message == response.json['error']['reason'] + + +@pytest.mark.service +@pytest.mark.integration +def test_add_file_view(svc_client_with_repo): + """Check adding of uploaded file to dataset.""" + svc_client, headers, project_id = svc_client_with_repo + content_type = headers.pop('Content-Type') + + response = svc_client.post( + '/cache/files-upload', + data=dict(file=(io.BytesIO(b'this is a test'), 'datafile1.txt'), ), + query_string={'override_existing': True}, + headers=headers + ) + + assert response + assert 200 == response.status_code + assert {'result'} == set(response.json.keys()) + assert 1 == len(response.json['result']['files']) + + file_id = response.json['result']['files'][0]['file_id'] + assert isinstance(uuid.UUID(file_id), uuid.UUID) + + payload = { + 'project_id': project_id, + 'dataset_name': '{0}'.format(uuid.uuid4().hex), + 'create_dataset': True, + 'files': [{ + 'file_id': file_id, + }, ] + } + headers['Content-Type'] = content_type + + response = svc_client.post( + '/datasets/add', + data=json.dumps(payload), + headers=headers, + ) + + assert response + + assert {'result'} == set(response.json.keys()) + assert {'dataset_name', 'project_id', + 'files'} == set(response.json['result'].keys()) + + assert 1 == len(response.json['result']['files']) + assert file_id == response.json['result']['files'][0]['file_id'] + + +@pytest.mark.service +@pytest.mark.integration +def test_list_datasets_view(svc_client_with_repo): + """Check listing of existing datasets.""" + svc_client, headers, project_id = svc_client_with_repo + + params = { + 'project_id': project_id, + } + + response = svc_client.get( + '/datasets/list', + query_string=params, + headers=headers, + ) + + assert response + + assert {'result'} == set(response.json.keys()) + assert {'datasets'} == set(response.json['result'].keys()) + assert 0 != len(response.json['result']['datasets']) + assert {'identifier', 'name', 'version', + 'created'} == set(response.json['result']['datasets'][0].keys()) + + +@pytest.mark.service +@pytest.mark.integration +def test_list_datasets_view_no_auth(svc_client_with_repo): + """Check listing of existing datasets with no auth.""" + svc_client, headers, project_id = svc_client_with_repo + + params = { + 'project_id': project_id, + } + + response = svc_client.get( + '/datasets/list', + query_string=params, + ) + + assert response + assert {'error'} == set(response.json.keys()) + + +@pytest.mark.service +@pytest.mark.integration +def test_create_and_list_datasets_view(svc_client_with_repo): + """Create and list created dataset.""" + svc_client, headers, project_id = svc_client_with_repo + + payload = { + 'project_id': project_id, + 'dataset_name': '{0}'.format(uuid.uuid4().hex), + } + + response = svc_client.post( + '/datasets/create', + data=json.dumps(payload), + headers=headers, + ) + + assert response + + assert {'result'} == set(response.json.keys()) + assert {'dataset_name'} == set(response.json['result'].keys()) + assert payload['dataset_name'] == response.json['result']['dataset_name'] + + params_list = { + 'project_id': project_id, + } + + response = svc_client.get( + '/datasets/list', + query_string=params_list, + headers=headers, + ) + + assert response + + assert {'result'} == set(response.json.keys()) + assert {'datasets'} == set(response.json['result'].keys()) + assert 0 != len(response.json['result']['datasets']) + assert {'identifier', 'name', 'version', + 'created'} == set(response.json['result']['datasets'][0].keys()) + + assert payload['dataset_name'] in [ + ds['name'] for ds in response.json['result']['datasets'] + ] + + +@pytest.mark.service +@pytest.mark.integration +def test_list_dataset_files(svc_client_with_repo): + """Check listing of dataset files""" + svc_client, headers, project_id = svc_client_with_repo + content_type = headers.pop('Content-Type') + + file_name = '{0}'.format(uuid.uuid4().hex) + response = svc_client.post( + '/cache/files-upload', + data=dict(file=(io.BytesIO(b'this is a test'), file_name), ), + query_string={'override_existing': True}, + headers=headers + ) + + assert response + assert 200 == response.status_code + + assert {'result'} == set(response.json.keys()) + assert 1 == len(response.json['result']['files']) + file_id = response.json['result']['files'][0]['file_id'] + assert isinstance(uuid.UUID(file_id), uuid.UUID) + + payload = { + 'project_id': project_id, + 'dataset_name': 'mydata', + 'files': [{ + 'file_id': file_id + }, ], + } + headers['Content-Type'] = content_type + + response = svc_client.post( + '/datasets/add', + data=json.dumps(payload), + headers=headers, + ) + + assert response + + assert {'result'} == set(response.json.keys()) + assert {'dataset_name', 'files', + 'project_id'} == set(response.json['result'].keys()) + assert file_id == response.json['result']['files'][0]['file_id'] + + params = { + 'project_id': project_id, + 'dataset_name': 'mydata', + } + + response = svc_client.get( + '/datasets/files-list', + query_string=params, + headers=headers, + ) + + assert response + + assert {'result'} == set(response.json.keys()) + assert {'dataset_name', 'files'} == set(response.json['result'].keys()) + + assert params['dataset_name'] == response.json['result']['dataset_name'] + assert file_name in [ + file['name'] for file in response.json['result']['files'] + ] + + +@pytest.mark.service +@pytest.mark.integration +def test_add_with_unpacked_archive(datapack_zip, svc_client_with_repo): + """Upload archive and add it to a dataset.""" + svc_client, headers, project_id = svc_client_with_repo + content_type = headers.pop('Content-Type') + + response = svc_client.post( + '/cache/files-upload', + data=dict( + file=(io.BytesIO(datapack_zip.read_bytes()), datapack_zip.name), + ), + query_string={ + 'unpack_archive': True, + 'override_existing': True, + }, + headers=headers + ) + + assert response + + assert 200 == response.status_code + assert {'result'} == set(response.json.keys()) + assert 2 == len(response.json['result']['files']) + + for file_ in response.json['result']['files']: + assert not file_['is_archive'] + assert not file_['unpack_archive'] + + file_id = file_['file_id'] + assert file_id + + file_ = response.json['result']['files'][0] + payload = { + 'project_id': project_id, + 'dataset_name': '{0}'.format(uuid.uuid4().hex), + } + + headers['Content-Type'] = content_type + response = svc_client.post( + '/datasets/create', + data=json.dumps(payload), + headers=headers, + ) + + assert response + + assert {'result'} == set(response.json.keys()) + assert {'dataset_name'} == set(response.json['result'].keys()) + assert payload['dataset_name'] == response.json['result']['dataset_name'] + + payload = { + 'project_id': project_id, + 'dataset_name': 'mydata', + 'files': [{ + 'file_id': file_['file_id'] + }, ] + } + + response = svc_client.post( + '/datasets/add', + data=json.dumps(payload), + headers=headers, + ) + + assert response + + assert {'result'} == set(response.json.keys()) + assert {'dataset_name', 'files', + 'project_id'} == set(response.json['result'].keys()) + assert file_['file_id'] == response.json['result']['files'][0]['file_id'] + + params = { + 'project_id': project_id, + 'dataset_name': 'mydata', + } + + response = svc_client.get( + '/datasets/files-list', + query_string=params, + headers=headers, + ) + + assert response + + assert {'result'} == set(response.json.keys()) + assert {'dataset_name', 'files'} == set(response.json['result'].keys()) + + assert params['dataset_name'] == response.json['result']['dataset_name'] + assert file_['file_name'] in [ + file['name'] for file in response.json['result']['files'] + ] + + +@pytest.mark.service +@pytest.mark.integration +def test_add_with_unpacked_archive_all(datapack_zip, svc_client_with_repo): + """Upload archive and add its contents to a dataset.""" + svc_client, headers, project_id = svc_client_with_repo + content_type = headers.pop('Content-Type') + + response = svc_client.post( + '/cache/files-upload', + data=dict( + file=(io.BytesIO(datapack_zip.read_bytes()), datapack_zip.name), + ), + query_string={ + 'unpack_archive': True, + 'override_existing': True, + }, + headers=headers + ) + + assert response + + assert 200 == response.status_code + assert {'result'} == set(response.json.keys()) + assert 2 == len(response.json['result']['files']) + + for file_ in response.json['result']['files']: + assert not file_['is_archive'] + assert not file_['unpack_archive'] + + file_id = file_['file_id'] + assert file_id + + files = [{ + 'file_id': file_['file_id'] + } for file_ in response.json['result']['files']] + + payload = { + 'project_id': project_id, + 'dataset_name': '{0}'.format(uuid.uuid4().hex), + } + + headers['Content-Type'] = content_type + response = svc_client.post( + '/datasets/create', + data=json.dumps(payload), + headers=headers, + ) + + assert response + + assert {'result'} == set(response.json.keys()) + assert {'dataset_name'} == set(response.json['result'].keys()) + assert payload['dataset_name'] == response.json['result']['dataset_name'] + + payload = { + 'project_id': project_id, + 'dataset_name': payload['dataset_name'], + 'files': files, + } + + response = svc_client.post( + '/datasets/add', + data=json.dumps(payload), + headers=headers, + ) + + assert response + + assert {'result'} == set(response.json.keys()) + assert {'dataset_name', 'files', + 'project_id'} == set(response.json['result'].keys()) + assert files == response.json['result']['files'] + + params = { + 'project_id': project_id, + 'dataset_name': payload['dataset_name'], + } + + response = svc_client.get( + '/datasets/files-list', + query_string=params, + headers=headers, + ) + + assert response + + assert {'result'} == set(response.json.keys()) + assert {'dataset_name', 'files'} == set(response.json['result'].keys()) + + assert params['dataset_name'] == response.json['result']['dataset_name'] + assert file_['file_name'] in [ + file['name'] for file in response.json['result']['files'] + ] diff --git a/tests/service/test_exceptions.py b/tests/service/test_exceptions.py new file mode 100644 index 0000000000..d5bd5b4c1f --- /dev/null +++ b/tests/service/test_exceptions.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2019 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service exception tests for all endpoints.""" +import pytest + +from renku.service.config import INVALID_HEADERS_ERROR_CODE + + +@pytest.mark.service +def test_allowed_methods_exc(service_allowed_endpoint): + """Check allowed methods for every endpoint.""" + methods, request, svc_client = service_allowed_endpoint + + method = request['allowed_method'] + if method == 'GET': # if GET remove sister method HEAD + methods.pop(method) + methods.pop('HEAD') + else: + methods.pop(method) + + for method, fn in methods.items(): + response = fn(request['url']) + assert 405 == response.status_code + + +@pytest.mark.service +def test_auth_headers_exc(service_allowed_endpoint): + """Check correct headers for every endpoint.""" + methods, request, svc_client = service_allowed_endpoint + + method = request['allowed_method'] + if method == 'GET': # if GET remove sister method HEAD + client_method = methods.pop(method) + methods.pop('HEAD') + else: + client_method = methods.pop(method) + + response = client_method( + request['url'], + headers=request['headers'], + ) + + assert 200 == response.status_code + assert INVALID_HEADERS_ERROR_CODE == response.json['error']['code'] + + err_message = 'user identification is incorrect or missing' + assert err_message == response.json['error']['reason'] From 84a4408da2da332d00ab16508b4797b96449ba37 Mon Sep 17 00:00:00 2001 From: sam Date: Tue, 3 Dec 2019 12:19:00 +0100 Subject: [PATCH 04/11] fix: addressed review points --- MANIFEST.in | 1 - conftest.py | 32 ++++++++++++++--------------- docker-compose.yml | 2 -- tests/service/test_cache_views.py | 6 +++--- tests/service/test_dataset_views.py | 4 ++-- 5 files changed, 21 insertions(+), 24 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index d77b546611..eb8dad3260 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -41,7 +41,6 @@ include babel.ini include brew.py include pytest.ini include snap/snapcraft.yaml -recursive-include renku *.json recursive-include .github CODEOWNERS recursive-include .travis *.sh recursive-include docs *.bat diff --git a/conftest.py b/conftest.py index 63f880b524..d66adeb244 100644 --- a/conftest.py +++ b/conftest.py @@ -514,38 +514,38 @@ def remote_project(data_repository, directory_tree): yield runner, project_path -@pytest.fixture(scope='function') -def dummy_datapack(): - """Creates dummy data folder.""" - temp_dir = tempfile.TemporaryDirectory() - - data_file_txt = Path(temp_dir.name) / Path('file.txt') - data_file_txt.write_text('my awesome data') - - data_file_csv = Path(temp_dir.name) / Path('file.csv') - data_file_csv.write_text('more,awesome,data') - - yield temp_dir +# @pytest.fixture(scope='function') +# def dummy_datapack(): +# """Creates dummy data folder.""" +# temp_dir = tempfile.TemporaryDirectory() +# +# data_file_txt = Path(temp_dir.name) / Path('file.txt') +# data_file_txt.write_text('my awesome data') +# +# data_file_csv = Path(temp_dir.name) / Path('file.csv') +# data_file_csv.write_text('more,awesome,data') +# +# yield temp_dir @pytest.fixture(scope='function') -def datapack_zip(dummy_datapack): +def datapack_zip(directory_tree): """Returns dummy data folder as a zip archive.""" from renku.core.utils.contexts import chdir workspace_dir = tempfile.TemporaryDirectory() with chdir(workspace_dir.name): - shutil.make_archive('datapack', 'zip', dummy_datapack.name) + shutil.make_archive('datapack', 'zip', directory_tree) yield Path(workspace_dir.name) / 'datapack.zip' @pytest.fixture(scope='function') -def datapack_tar(dummy_datapack): +def datapack_tar(directory_tree): """Returns dummy data folder as a tar archive.""" from renku.core.utils.contexts import chdir workspace_dir = tempfile.TemporaryDirectory() with chdir(workspace_dir.name): - shutil.make_archive('datapack', 'tar', dummy_datapack.name) + shutil.make_archive('datapack', 'tar', directory_tree) yield Path(workspace_dir.name) / 'datapack.tar' diff --git a/docker-compose.yml b/docker-compose.yml index 96436d1b24..140132e5a8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,8 +3,6 @@ version: '3' services: redis: image: redis:5.0.3-alpine - ports: - - "6379:6379" renku-svc: image: renku-svc:latest diff --git a/tests/service/test_cache_views.py b/tests/service/test_cache_views.py index dcc066cf64..5d39dec993 100644 --- a/tests/service/test_cache_views.py +++ b/tests/service/test_cache_views.py @@ -462,7 +462,7 @@ def test_upload_zip_unpack_archive(datapack_zip, svc_client_with_repo): assert 200 == response.status_code assert {'result'} == set(response.json.keys()) - assert 2 == len(response.json['result']['files']) + assert 3 == len(response.json['result']['files']) for file_ in response.json['result']['files']: assert not file_['is_archive'] @@ -520,7 +520,7 @@ def test_upload_tar_unpack_archive(datapack_tar, svc_client_with_repo): assert 200 == response.status_code assert {'result'} == set(response.json.keys()) - assert 2 == len(response.json['result']['files']) + assert 3 == len(response.json['result']['files']) for file_ in response.json['result']['files']: assert not file_['is_archive'] @@ -579,7 +579,7 @@ def test_field_upload_resp_fields(datapack_tar, svc_client_with_repo): assert 200 == response.status_code assert {'result'} == set(response.json.keys()) - assert 2 == len(response.json['result']['files']) + assert 3 == len(response.json['result']['files']) assert { 'content_type', 'file_id', diff --git a/tests/service/test_dataset_views.py b/tests/service/test_dataset_views.py index 60780daa6b..6a8fb98245 100644 --- a/tests/service/test_dataset_views.py +++ b/tests/service/test_dataset_views.py @@ -381,7 +381,7 @@ def test_add_with_unpacked_archive(datapack_zip, svc_client_with_repo): assert 200 == response.status_code assert {'result'} == set(response.json.keys()) - assert 2 == len(response.json['result']['files']) + assert 3 == len(response.json['result']['files']) for file_ in response.json['result']['files']: assert not file_['is_archive'] @@ -475,7 +475,7 @@ def test_add_with_unpacked_archive_all(datapack_zip, svc_client_with_repo): assert 200 == response.status_code assert {'result'} == set(response.json.keys()) - assert 2 == len(response.json['result']['files']) + assert 3 == len(response.json['result']['files']) for file_ in response.json['result']['files']: assert not file_['is_archive'] From 5d0859ee6167502496517239c2b0decfb5862859 Mon Sep 17 00:00:00 2001 From: sam Date: Tue, 3 Dec 2019 16:42:40 +0100 Subject: [PATCH 05/11] chore: added travis encrypted oauth token --- .travis.yml | 333 ++++++++++++++---------------- conftest.py | 20 +- tests/service/test_cache_views.py | 9 +- 3 files changed, 162 insertions(+), 200 deletions(-) diff --git a/.travis.yml b/.travis.yml index 70eaaa17be..39b26c9563 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,200 +18,175 @@ notifications: email: false - git: depth: false - language: python - matrix: fast_finish: true - cache: - - pip - +- pip env: + matrix: - REQUIREMENTS=lowest - REQUIREMENTS=release - + global: + secure: qLF8toEnN3JECO8sZlpE7Q5zPZhVt0y+kZtB8Vd/9kICdQkWh+/NKD6Mw0W+BW9hf+BkaCK2LEvCbeuSNg5lFr1aGZDvveMf8t3KkI1Aem+L4ewSSEbxekVtPiXsRf+Am6WOxZftntCo415aPxUYD8Ny+kZsb0DoJ4sxodAGmitUeFEo9f8bYGurDMAH7OC7AukYnRA33x8OVImU4G5uUML9z8q1pGUXZIldEucyDb0207zxn7UBwQCfhQm+HHTovOmZO3gvZvT5AJU3IQaiu7ePoBiK+M3Vb4cyHl4FlWE+5NZMpk/c9aoFBpkO5aC9QrCDCfiq7lSULL7Gkte+uWnjBm7jJH74fLe4Ryclfodb8vKHpC7fYCwfOJHXjHRr5KDPSG/1KMCTv7r4sQ6GJCnN01bDuW64IV7VK+QJwoZZOJx4J3dFMeCJdB/tOtevmDE5bAFGoV0Ycr03g9N0aHbdr0me6vWPksqR7RVEfRsX5rXPARUZ+7kWkt2MFqgG/L8orPCzyxZtqrRPtbsltK1ZmJUR69v9Tb9y+EJxB2MGUkUs9DUOr99pAlJvdx29AOzHEK45IySTdYfxjeCMCw6J/1UlZRaYjW4mj5ag0R5cnS0907w864dp7FLV9wJ2Cg4iG8WLrt5GKUMMSpac9Y/Gwaf0vExciwK60cUewMU= dist: xenial - python: - - "3.5" - - "3.6" - - "3.7" - +- '3.5' +- '3.6' +- '3.7' stages: - - name: docs - - name: test - - name: integration - if: branch = master AND (type != pull_request) - - name: test OSX - if: branch = master AND (type != pull_request) - - name: publish 🐍 - if: type = push AND (branch = master OR tag IS present) - - name: brew 🍺 - if: type = push AND tag IS present AND tag =~ /^v\d\.\d\.\d$/ - +- name: docs +- name: test +- name: integration + if: branch = master AND (type != pull_request) +- name: test OSX + if: branch = master AND (type != pull_request) +- name: "publish \U0001F40D" + if: type = push AND (branch = master OR tag IS present) +- name: "brew \U0001F37A" + if: type = push AND tag IS present AND tag =~ /^v\d\.\d\.\d$/ before_install: - - git fetch --tags - - git config --global --add user.name "Renku @ SDSC" - - git config --global --add user.email "renku@datascience.ch" - - if [[ $TRAVIS_OS_NAME == 'linux' ]]; then - sudo apt-get update; - sudo apt-get -y install shellcheck; - travis_retry python -m pip install --upgrade six pip setuptools py; - travis_retry python -m pip install twine wheel coveralls requirements-builder; - requirements-builder -e all --level=min setup.py > .travis-lowest-requirements.txt; - requirements-builder -e all --level=pypi setup.py > .travis-release-requirements.txt; - requirements-builder -e all --level=dev --req requirements-devel.txt setup.py > .travis-devel-requirements.txt; - requirements-builder -e nodocs --level=min setup.py > .travis-lowest-requirements-nodocs.txt; - requirements-builder -e nodocs --level=pypi setup.py > .travis-release-requirements-nodocs.txt; - requirements-builder -e nodocs --level=dev --req requirements-devel.txt setup.py > .travis-devel-requirements-nodocs.txt; - elif [[ $TRAVIS_OS_NAME == 'osx' ]]; then - ulimit -n 1024; - brew update; - brew upgrade -v python; - brew unlink python; - brew link python; - brew install -v git-lfs jq node pipenv shellcheck; - travis_wait brew upgrade node; - fi - +- git fetch --tags +- git config --global --add user.name "Renku @ SDSC" +- git config --global --add user.email "renku@datascience.ch" +- if [[ $TRAVIS_OS_NAME == 'linux' ]]; then sudo apt-get update; sudo apt-get -y install + shellcheck; travis_retry python -m pip install --upgrade six pip setuptools py; + travis_retry python -m pip install twine wheel coveralls requirements-builder; requirements-builder + -e all --level=min setup.py > .travis-lowest-requirements.txt; requirements-builder + -e all --level=pypi setup.py > .travis-release-requirements.txt; requirements-builder + -e all --level=dev --req requirements-devel.txt setup.py > .travis-devel-requirements.txt; + requirements-builder -e nodocs --level=min setup.py > .travis-lowest-requirements-nodocs.txt; + requirements-builder -e nodocs --level=pypi setup.py > .travis-release-requirements-nodocs.txt; + requirements-builder -e nodocs --level=dev --req requirements-devel.txt setup.py + > .travis-devel-requirements-nodocs.txt; elif [[ $TRAVIS_OS_NAME == 'osx' ]]; then + ulimit -n 1024; brew update; brew upgrade -v python; brew unlink python; brew link + python; brew install -v git-lfs jq node pipenv shellcheck; travis_wait brew upgrade + node; fi install: - - if [[ $TRAVIS_OS_NAME == 'linux' ]]; then - travis_retry python -m pip install -r .travis-${REQUIREMENTS}-requirements-nodocs.txt; - travis_retry python -m pip install -e .[nodocs]; - elif [[ $TRAVIS_OS_NAME == 'osx' ]]; then - travis_retry pipenv install --deploy; - source "$(pipenv --venv)/bin/activate"; - travis_retry pip install -e .[nodocs]; - fi - +- if [[ $TRAVIS_OS_NAME == 'linux' ]]; then travis_retry python -m pip install -r + .travis-${REQUIREMENTS}-requirements-nodocs.txt; travis_retry python -m pip install + -e .[nodocs]; elif [[ $TRAVIS_OS_NAME == 'osx' ]]; then travis_retry pipenv install + --deploy; source "$(pipenv --venv)/bin/activate"; travis_retry pip install -e .[nodocs]; + fi script: - - "./run-tests.sh -t -s" - +- "./run-tests.sh -t -s" after_success: - - coveralls - +- coveralls jobs: include: - - stage: docs - os: linux - dist: xenial - language: python - env: - - REQUIREMENTS=lowest - - REQUIREMENTS=release - install: - - travis_retry python -m pip install -r .travis-${REQUIREMENTS}-requirements-all.txt; - travis_retry python -m pip install -e .[all]; - script: ./run-tests.sh -d - - stage: integration - os: linux - dist: xenial - language: python - env: - - REQUIREMENTS=release - script: pytest -m integration -v - - stage: integration - python: "3.6" - os: linux - dist: xenial - language: python - env: - - REQUIREMENTS=release - script: pytest -m integration -v - - stage: integration - python: "3.7" - os: linux - dist: xenial - language: python - env: - - REQUIREMENTS=release - script: pytest -m integration -v - - stage: integration - os: linux - dist: xenial - language: python - env: - - REQUIREMENTS=lowest - script: pytest -m integration -v - - stage: integration - python: "3.6" - os: linux - dist: xenial - language: python - env: - - REQUIREMENTS=lowest - script: pytest -m integration -v - - stage: integration - python: "3.7" - os: linux - dist: xenial - language: python - env: - - REQUIREMENTS=lowest - script: pytest -m integration -v - - stage: test OSX - language: generic - sudo: true - os: osx - osx_image: xcode11.2 - - stage: test OSX - language: generic - sudo: true - os: osx - osx_image: xcode10.1 - - - stage: publish 🐍 - python: 3.6 - script: echo "Publishing on PyPI.io ..." - before_deploy: - if [[ -z $TRAVIS_TAG ]]; then - export TRAVIS_TAG=$(renku --version) && - git tag $TRAVIS_TAG; - fi - deploy: - - provider: pypi - user: - secure: "RPxGYNL+N6LQy1/TbXCFy9IDgZ05u/Qj6my/p1hSoBWG304se28njZ0zpTv5AGZF8G3dBeVjYbf4s+ytwUPoxK+1fBWlnnSuw4QUXWf339rzcMU4dDO2QX8/SvMDZFbJdE/1/6arlbXa8dTZobm1V0Q3AUTkl2AwXIg9jCaRcYb6n9lASRIiR1M2BK/+FTJG2qywB9kSyQ3giuobwwhQa4CAwDg5MoqL5KLFm2CxejvS0vpleaWqA/LJxeHTQqqd0CIk6V2NBETZ6V78MqdISai88nRafijT0iQ5LSWsy7R6CCpK7OFjHnvA7hGSYzs/BRpdABAk5a2yFbKKZErXoLvatjflMlj2OhHy/0Hlv6xEt1db1pwnjQQIiS62R/Gpx4DZAO8hGp6pT9g9xiifzlj4km9iOD4GY1g+A5A+ssEneBTvExJja4yAqJzAVu+XVDVqxVj+MOmpIcQkzT983+cVceoeczJ61sDuftQaAgcVqQACRE02fLszEtSJVFaq3vKu8dX2eMdiCk7GLdqNF9kfygagNC8eja6Yvr+Ft8kTwrjTBMC/D3xC584I8OTzmpNE/tfZHppfhiKXoU+FySdIGCPcSTGKUgljiz3sFk1JjjEBkGqBLAMaD8l5FsgQqR4zO/2IiwSst1Wx8owF+tiLlerDAC9b/ZFcqDpUab4=" - password: - secure: "f8f2175dg1FUt4mZCQ87tgwwUrFoxQ5CTcZkRnlCXJqagSMk9VmjR8/XXDE5cW48JPG0qdKQdvBtC81NCq+4MqC20HI9VtOdZWeT6Jej90XOZ742osC3pdIGMF4wrsd7+fs1AZkbzzjgB7FsJ42qW6rMa3vP4mXB4GJEel453Fs3Fw8tnR4PZ2ikOJ9fcmtMensjxh9oNMyAIMkYVGim5bWtpkpI1cddeThDBEKurj1IWEMonQw4lR2yLwZTEP6F/b43Cy4aNy+YkdJzJbM0KMJASLeeu8SxNTE7JCqrYc4irU3AzHzzy/FNVGdiw0F10kbK+yI7cPUqWfeoVjwqBJe8Hr0CXNNCsEYkjXkY9PE2m2x10z2UrTy+O1dNo/8sDoKvZBChuAPPtxp2I7/KujECWjzFvMunHEk3K940ZeSMo90xHKQllmA7umquNcPTXiM2l4nNHtolh5W2HdanvsEFhkY2Y2o9sRIytOa5bM+iS9RCL5GsZwLgGKdjfuqk0GF6KK4AIgi7WKtVM73vM7HQaAVRpyUWZ/h8Vu5IRdkORC20WwHZ/Jg6pUy3pkN1VlcKE5uanaPik/npq/uCGe9YC2bh5IoclmqdJUHrkzFPb+f8wRBvbof0zU1B6UMSdiug5oDk3a0Q8kk2AppfjTs7x8NHi3KYXdUphi9HrWQ=" - distributions: "sdist bdist_wheel" - on: - all_branches: true - # push the dev tag to github - - provider: releases - api_key: ${GITHUB_TOKEN} - on: - all_branches: true - tags: false - - - stage: brew 🍺 - language: generic - sudo: true - os: osx - osx_image: xcode11.2 - before_install: brew -v install jq - install: python -m pip install requests - script: - - echo PY_BREW_VERSION=$(python setup.py --version) - - make -e PY_BREW_VERSION=$(python setup.py --version) brew-commit-formula brew-build-bottle brew-commit-bottle - deploy: - - provider: releases - api_key: ${GITHUB_TOKEN} - file_glob: true - file: "*.bottle.*" - skip_cleanup: true - on: - all_branches: true - - provider: pages - skip-cleanup: true - github-token: ${GITHUB_TOKEN} - repo: swissdatasciencecenter/homebrew-renku - target-branch: master - keep-history: true - local-dir: /usr/local/Homebrew/Library/Taps/swissdatasciencecenter/homebrew-renku/ - on: - all_branches: true + - stage: docs + os: linux + dist: xenial + language: python + env: + - REQUIREMENTS=lowest + - REQUIREMENTS=release + install: + - travis_retry python -m pip install -r .travis-${REQUIREMENTS}-requirements-all.txt; + travis_retry python -m pip install -e .[all]; + script: "./run-tests.sh -d" + - stage: integration + os: linux + dist: xenial + language: python + env: + - REQUIREMENTS=release + script: pytest -m integration -v + - stage: integration + python: '3.6' + os: linux + dist: xenial + language: python + env: + - REQUIREMENTS=release + script: pytest -m integration -v + - stage: integration + python: '3.7' + os: linux + dist: xenial + language: python + env: + - REQUIREMENTS=release + script: pytest -m integration -v + - stage: integration + os: linux + dist: xenial + language: python + env: + - REQUIREMENTS=lowest + script: pytest -m integration -v + - stage: integration + python: '3.6' + os: linux + dist: xenial + language: python + env: + - REQUIREMENTS=lowest + script: pytest -m integration -v + - stage: integration + python: '3.7' + os: linux + dist: xenial + language: python + env: + - REQUIREMENTS=lowest + script: pytest -m integration -v + - stage: test OSX + language: generic + sudo: true + os: osx + osx_image: xcode11.2 + - stage: test OSX + language: generic + sudo: true + os: osx + osx_image: xcode10.1 + - stage: "publish \U0001F40D" + python: 3.6 + script: echo "Publishing on PyPI.io ..." + before_deploy: if [[ -z $TRAVIS_TAG ]]; then export TRAVIS_TAG=$(renku --version) + && git tag $TRAVIS_TAG; fi + deploy: + - provider: pypi + user: + secure: RPxGYNL+N6LQy1/TbXCFy9IDgZ05u/Qj6my/p1hSoBWG304se28njZ0zpTv5AGZF8G3dBeVjYbf4s+ytwUPoxK+1fBWlnnSuw4QUXWf339rzcMU4dDO2QX8/SvMDZFbJdE/1/6arlbXa8dTZobm1V0Q3AUTkl2AwXIg9jCaRcYb6n9lASRIiR1M2BK/+FTJG2qywB9kSyQ3giuobwwhQa4CAwDg5MoqL5KLFm2CxejvS0vpleaWqA/LJxeHTQqqd0CIk6V2NBETZ6V78MqdISai88nRafijT0iQ5LSWsy7R6CCpK7OFjHnvA7hGSYzs/BRpdABAk5a2yFbKKZErXoLvatjflMlj2OhHy/0Hlv6xEt1db1pwnjQQIiS62R/Gpx4DZAO8hGp6pT9g9xiifzlj4km9iOD4GY1g+A5A+ssEneBTvExJja4yAqJzAVu+XVDVqxVj+MOmpIcQkzT983+cVceoeczJ61sDuftQaAgcVqQACRE02fLszEtSJVFaq3vKu8dX2eMdiCk7GLdqNF9kfygagNC8eja6Yvr+Ft8kTwrjTBMC/D3xC584I8OTzmpNE/tfZHppfhiKXoU+FySdIGCPcSTGKUgljiz3sFk1JjjEBkGqBLAMaD8l5FsgQqR4zO/2IiwSst1Wx8owF+tiLlerDAC9b/ZFcqDpUab4= + password: + secure: f8f2175dg1FUt4mZCQ87tgwwUrFoxQ5CTcZkRnlCXJqagSMk9VmjR8/XXDE5cW48JPG0qdKQdvBtC81NCq+4MqC20HI9VtOdZWeT6Jej90XOZ742osC3pdIGMF4wrsd7+fs1AZkbzzjgB7FsJ42qW6rMa3vP4mXB4GJEel453Fs3Fw8tnR4PZ2ikOJ9fcmtMensjxh9oNMyAIMkYVGim5bWtpkpI1cddeThDBEKurj1IWEMonQw4lR2yLwZTEP6F/b43Cy4aNy+YkdJzJbM0KMJASLeeu8SxNTE7JCqrYc4irU3AzHzzy/FNVGdiw0F10kbK+yI7cPUqWfeoVjwqBJe8Hr0CXNNCsEYkjXkY9PE2m2x10z2UrTy+O1dNo/8sDoKvZBChuAPPtxp2I7/KujECWjzFvMunHEk3K940ZeSMo90xHKQllmA7umquNcPTXiM2l4nNHtolh5W2HdanvsEFhkY2Y2o9sRIytOa5bM+iS9RCL5GsZwLgGKdjfuqk0GF6KK4AIgi7WKtVM73vM7HQaAVRpyUWZ/h8Vu5IRdkORC20WwHZ/Jg6pUy3pkN1VlcKE5uanaPik/npq/uCGe9YC2bh5IoclmqdJUHrkzFPb+f8wRBvbof0zU1B6UMSdiug5oDk3a0Q8kk2AppfjTs7x8NHi3KYXdUphi9HrWQ= + distributions: sdist bdist_wheel + on: + all_branches: true + - provider: releases + api_key: "${GITHUB_TOKEN}" + on: + all_branches: true + tags: false + - stage: "brew \U0001F37A" + language: generic + sudo: true + os: osx + osx_image: xcode11.2 + before_install: brew -v install jq + install: python -m pip install requests + script: + - echo PY_BREW_VERSION=$(python setup.py --version) + - make -e PY_BREW_VERSION=$(python setup.py --version) brew-commit-formula brew-build-bottle + brew-commit-bottle + deploy: + - provider: releases + api_key: "${GITHUB_TOKEN}" + file_glob: true + file: "*.bottle.*" + skip_cleanup: true + on: + all_branches: true + - provider: pages + skip-cleanup: true + github-token: "${GITHUB_TOKEN}" + repo: swissdatasciencecenter/homebrew-renku + target-branch: master + keep-history: true + local-dir: "/usr/local/Homebrew/Library/Taps/swissdatasciencecenter/homebrew-renku/" + on: + all_branches: true diff --git a/conftest.py b/conftest.py index d66adeb244..03b9250965 100644 --- a/conftest.py +++ b/conftest.py @@ -514,27 +514,13 @@ def remote_project(data_repository, directory_tree): yield runner, project_path -# @pytest.fixture(scope='function') -# def dummy_datapack(): -# """Creates dummy data folder.""" -# temp_dir = tempfile.TemporaryDirectory() -# -# data_file_txt = Path(temp_dir.name) / Path('file.txt') -# data_file_txt.write_text('my awesome data') -# -# data_file_csv = Path(temp_dir.name) / Path('file.csv') -# data_file_csv.write_text('more,awesome,data') -# -# yield temp_dir - - @pytest.fixture(scope='function') def datapack_zip(directory_tree): """Returns dummy data folder as a zip archive.""" from renku.core.utils.contexts import chdir workspace_dir = tempfile.TemporaryDirectory() with chdir(workspace_dir.name): - shutil.make_archive('datapack', 'zip', directory_tree) + shutil.make_archive('datapack', 'zip', str(directory_tree)) yield Path(workspace_dir.name) / 'datapack.zip' @@ -545,7 +531,7 @@ def datapack_tar(directory_tree): from renku.core.utils.contexts import chdir workspace_dir = tempfile.TemporaryDirectory() with chdir(workspace_dir.name): - shutil.make_archive('datapack', 'tar', directory_tree) + shutil.make_archive('datapack', 'tar', str(directory_tree)) yield Path(workspace_dir.name) / 'datapack.tar' @@ -586,7 +572,7 @@ def svc_client_with_repo(svc_client, mock_redis): 'Renku-User-Id': 'b4b4de0eda0f471ab82702bd5c367fa7', 'Renku-User-FullName': 'Just Sam', 'Renku-User-Email': 'contact@justsam.io', - 'Authorization': 'Bearer LkoLiyLqnhMCAa4or5qa', + 'Authorization': 'Bearer {0}'.format(os.getenv('IT_OAUTH_GIT_TOKEN')), } payload = {'git_url': remote_url} diff --git a/tests/service/test_cache_views.py b/tests/service/test_cache_views.py index 5d39dec993..57984b982b 100644 --- a/tests/service/test_cache_views.py +++ b/tests/service/test_cache_views.py @@ -18,6 +18,7 @@ """Renku service cache view tests.""" import io import json +import os import uuid import pytest @@ -27,7 +28,7 @@ INVALID_PARAMS_ERROR_CODE REMOTE_URL = 'https://dev.renku.ch/gitlab/contact/integration-tests' -PERSONAL_ACCESS_TOKEN = 'LkoLiyLqnhMCAa4or5qa' +IT_GIT_ACCESS_TOKEN = os.getenv('IT_OAUTH_GIT_TOKEN') @pytest.mark.service @@ -278,7 +279,7 @@ def test_clone_projects_with_auth(svc_client): 'Renku-User-Id': '{0}'.format(uuid.uuid4().hex), 'Renku-User-FullName': 'Just Sam', 'Renku-User-Email': 'contact@justsam.io', - 'Authorization': 'Bearer {0}'.format(PERSONAL_ACCESS_TOKEN), + 'Authorization': 'Bearer {0}'.format(IT_GIT_ACCESS_TOKEN), } payload = { @@ -304,7 +305,7 @@ def test_clone_projects_multiple(svc_client): 'Renku-User-Id': '{0}'.format(uuid.uuid4().hex), 'Renku-User-FullName': 'Just Sam', 'Renku-User-Email': 'contact@justsam.io', - 'Authorization': 'Bearer {0}'.format(PERSONAL_ACCESS_TOKEN), + 'Authorization': 'Bearer {0}'.format(IT_GIT_ACCESS_TOKEN), } payload = { @@ -364,7 +365,7 @@ def test_clone_projects_list_view_errors(svc_client): 'Renku-User-Id': '{0}'.format(uuid.uuid4().hex), 'Renku-User-FullName': 'Just Sam', 'Renku-User-Email': 'contact@justsam.io', - 'Authorization': 'Bearer {0}'.format(PERSONAL_ACCESS_TOKEN), + 'Authorization': 'Bearer {0}'.format(IT_GIT_ACCESS_TOKEN), } payload = { From 763193b4b628117b51e83ea00ecddadc0a6c9f41 Mon Sep 17 00:00:00 2001 From: sam Date: Wed, 4 Dec 2019 18:03:21 +0100 Subject: [PATCH 06/11] fix: cleanup of user vs user_id --- renku/service/cache/files.py | 2 +- renku/service/cache/projects.py | 2 +- renku/service/serializers/headers.py | 2 +- renku/service/utils/__init__.py | 10 ++++++---- renku/service/views/cache.py | 23 +++++++++++++---------- renku/service/views/datasets.py | 10 +++++----- tests/service/test_cache_views.py | 1 + 7 files changed, 28 insertions(+), 22 deletions(-) diff --git a/renku/service/cache/files.py b/renku/service/cache/files.py index ab8240c79e..cb344e1ed5 100644 --- a/renku/service/cache/files.py +++ b/renku/service/cache/files.py @@ -26,7 +26,7 @@ class FileManagementCache(BaseCache): def files_cache_key(self, user): """Construct cache key based on user and files suffix.""" - return '{0}_{1}'.format(user, self.FILES_SUFFIX) + return '{0}_{1}'.format(user['user_id'], self.FILES_SUFFIX) def set_file(self, user, file_id, metadata): """Cache file metadata under user hash set.""" diff --git a/renku/service/cache/projects.py b/renku/service/cache/projects.py index 21f10e9a8a..ebd092cc4e 100644 --- a/renku/service/cache/projects.py +++ b/renku/service/cache/projects.py @@ -26,7 +26,7 @@ class ProjectManagementCache(BaseCache): def projects_cache_key(self, user): """Construct cache key based on user and projects suffix.""" - return '{0}_{1}'.format(user, self.PROJECTS_SUFFIX) + return '{0}_{1}'.format(user['user_id'], self.PROJECTS_SUFFIX) def set_project(self, user, project_id, metadata): """Cache project metadata under user hash set.""" diff --git a/renku/service/serializers/headers.py b/renku/service/serializers/headers.py index f1ab332fb0..c67abdf179 100644 --- a/renku/service/serializers/headers.py +++ b/renku/service/serializers/headers.py @@ -23,7 +23,7 @@ class UserIdentityHeaders(Schema): """User identity schema.""" - uid = fields.String(required=True, data_key='renku-user-id') + user_id = fields.String(required=True, data_key='renku-user-id') fullname = fields.String(data_key='renku-user-fullname') email = fields.String(data_key='renku-user-email') token = fields.String(data_key='authorization') diff --git a/renku/service/utils/__init__.py b/renku/service/utils/__init__.py index c5d87b539b..d731a1ba3e 100644 --- a/renku/service/utils/__init__.py +++ b/renku/service/utils/__init__.py @@ -23,23 +23,25 @@ def make_project_path(user, project): """Construct full path for cached project.""" - valid_user = user and 'uid' in user + valid_user = user and 'user_id' in user valid_project = project and 'owner' in project and 'name' in project if valid_user and valid_project: return ( - CACHE_PROJECTS_PATH / user['uid'] / project['owner'] / + CACHE_PROJECTS_PATH / user['user_id'] / project['owner'] / project['name'] ) def make_file_path(user, cached_file): """Construct full path for cache file.""" - valid_user = user and 'uid' in user + valid_user = user and 'user_id' in user valid_file = cached_file and 'file_name' in cached_file if valid_user and valid_file: - return CACHE_UPLOADS_PATH / user['uid'] / cached_file['relative_path'] + return ( + CACHE_UPLOADS_PATH / user['user_id'] / cached_file['relative_path'] + ) def repo_sync(repo_path, remote_names=('origin', )): diff --git a/renku/service/views/cache.py b/renku/service/views/cache.py index 983c42d534..d0686dceaa 100644 --- a/renku/service/views/cache.py +++ b/renku/service/views/cache.py @@ -59,8 +59,7 @@ def list_uploaded_files_view(user, cache): """List uploaded files ready to be added to projects.""" files = [ - f for f in cache.get_files(user['uid']) - if make_file_path(user, f).exists() + f for f in cache.get_files(user) if make_file_path(user, f).exists() ] response = FileListResponseRPC().load({ @@ -97,7 +96,7 @@ def upload_file_view(user, cache): } response_builder.update(FileUploadRequest().load(request.args)) - user_cache_dir = CACHE_UPLOADS_PATH / user['uid'] + user_cache_dir = CACHE_UPLOADS_PATH / user['user_id'] user_cache_dir.mkdir(exist_ok=True) file_path = user_cache_dir / file.filename @@ -137,7 +136,11 @@ def upload_file_view(user, cache): 'file_name': file_.name, 'file_size': os.stat(str(file_path)).st_size, 'relative_path': - str(file_.relative_to(CACHE_UPLOADS_PATH / user['uid'])) + str( + file_.relative_to( + CACHE_UPLOADS_PATH / user['user_id'] + ) + ) } files.append(FileUploadContext().load(file_obj, unknown=EXCLUDE)) @@ -145,7 +148,7 @@ def upload_file_view(user, cache): else: response_builder['file_size'] = os.stat(str(file_path)).st_size response_builder['relative_path'] = str( - file_path.relative_to(CACHE_UPLOADS_PATH / user['uid']) + file_path.relative_to(CACHE_UPLOADS_PATH / user['user_id']) ) files.append( @@ -155,7 +158,7 @@ def upload_file_view(user, cache): response = FileUploadResponseRPC().load({ 'result': FileUploadResponse().load({'files': files}) }) - cache.set_files(user['uid'], files) + cache.set_files(user, files) return jsonify(response) @@ -191,9 +194,9 @@ def project_clone(user, cache): if local_path.exists(): shutil.rmtree(str(local_path)) - for project in cache.get_projects(user['uid']): + for project in cache.get_projects(user): if project['git_url'] == ctx['git_url']: - cache.invalidate_project(user['uid'], project['project_id']) + cache.invalidate_project(user, project['project_id']) local_path.mkdir(parents=True, exist_ok=True) renku_clone( @@ -206,7 +209,7 @@ def project_clone(user, cache): 'user.email': ctx['email'], } ) - cache.set_project(user['uid'], ctx['project_id'], ctx) + cache.set_project(user, ctx['project_id'], ctx) response = ProjectCloneResponseRPC().load({ 'result': ProjectCloneResponse().load(ctx, unknown=EXCLUDE) @@ -232,7 +235,7 @@ def project_clone(user, cache): @requires_identity def list_projects_view(user, cache): """List cached projects.""" - projects = cache.get_projects(user['uid']) + projects = cache.get_projects(user) projects = [ ProjectCloneResponse().load(p, unknown=EXCLUDE) for p in projects if make_project_path(user, p).exists() diff --git a/renku/service/views/datasets.py b/renku/service/views/datasets.py index 3984ee8877..2fb71f744f 100644 --- a/renku/service/views/datasets.py +++ b/renku/service/views/datasets.py @@ -59,7 +59,7 @@ def list_datasets_view(user, cache): """List all datasets in project.""" req = DatasetListRequest().load(request.args) - project = cache.get_project(user['uid'], req['project_id']) + project = cache.get_project(user, req['project_id']) project_path = make_project_path(user, project) if not project_path: @@ -98,7 +98,7 @@ def list_datasets_view(user, cache): def list_dataset_files_view(user, cache): """List files in a dataset.""" ctx = DatasetFilesListRequest().load(request.args) - project = cache.get_project(user['uid'], ctx['project_id']) + project = cache.get_project(user, ctx['project_id']) project_path = make_project_path(user, project) if not project_path: @@ -143,7 +143,7 @@ def list_dataset_files_view(user, cache): def add_file_to_dataset_view(user, cache): """Add uploaded file to cloned repository.""" ctx = DatasetAddRequest().load(request.json) - project = cache.get_project(user['uid'], ctx['project_id']) + project = cache.get_project(user, ctx['project_id']) project_path = make_project_path(user, project) if not project_path: return jsonify( @@ -155,7 +155,7 @@ def add_file_to_dataset_view(user, cache): local_paths = [] for file_ in ctx['files']: - file = cache.get_file(user['uid'], file_['file_id']) + file = cache.get_file(user, file_['file_id']) local_path = make_file_path(user, file) if not local_path or not local_path.exists(): return jsonify( @@ -206,7 +206,7 @@ def add_file_to_dataset_view(user, cache): def create_dataset_view(user, cache): """Create a new dataset in a project.""" ctx = DatasetCreateRequest().load(request.json) - project = cache.get_project(user['uid'], ctx['project_id']) + project = cache.get_project(user, ctx['project_id']) project_path = make_project_path(user, project) if not project_path: diff --git a/tests/service/test_cache_views.py b/tests/service/test_cache_views.py index 57984b982b..55e972e5b3 100644 --- a/tests/service/test_cache_views.py +++ b/tests/service/test_cache_views.py @@ -317,6 +317,7 @@ def test_clone_projects_multiple(svc_client): ) assert response + assert {'result'} == set(response.json.keys()) project_ids.append(response.json['result']) From 529e42e1825909b56f641649bc204e72d9352d65 Mon Sep 17 00:00:00 2001 From: sam Date: Mon, 16 Dec 2019 15:49:09 +0100 Subject: [PATCH 07/11] chore: method naming convention update --- conftest.py | 18 ++++----- renku/service/views/cache.py | 8 ++-- renku/service/views/datasets.py | 8 ++-- tests/service/test_cache_views.py | 62 ++++++++++++++--------------- tests/service/test_dataset_views.py | 44 ++++++++++---------- 5 files changed, 70 insertions(+), 70 deletions(-) diff --git a/conftest.py b/conftest.py index 03b9250965..5c6d301204 100644 --- a/conftest.py +++ b/conftest.py @@ -578,7 +578,7 @@ def svc_client_with_repo(svc_client, mock_redis): payload = {'git_url': remote_url} response = svc_client.post( - '/cache/project-clone', + '/cache.project_clone', data=json.dumps(payload), headers=headers, ) @@ -595,7 +595,7 @@ def svc_client_with_repo(svc_client, mock_redis): @pytest.fixture( params=[ { - 'url': '/cache/files-list', + 'url': '/cache.files_list', 'allowed_method': 'GET', 'headers': { 'Content-Type': 'application/json', @@ -603,12 +603,12 @@ def svc_client_with_repo(svc_client, mock_redis): } }, { - 'url': '/cache/files-upload', + 'url': '/cache.files_upload', 'allowed_method': 'POST', 'headers': {} }, { - 'url': '/cache/project-clone', + 'url': '/cache.project_clone', 'allowed_method': 'POST', 'headers': { 'Content-Type': 'application/json', @@ -616,7 +616,7 @@ def svc_client_with_repo(svc_client, mock_redis): } }, { - 'url': '/cache/project-list', + 'url': '/cache.project_list', 'allowed_method': 'GET', 'headers': { 'Content-Type': 'application/json', @@ -624,7 +624,7 @@ def svc_client_with_repo(svc_client, mock_redis): } }, { - 'url': '/datasets/add', + 'url': '/datasets.add', 'allowed_method': 'POST', 'headers': { 'Content-Type': 'application/json', @@ -632,7 +632,7 @@ def svc_client_with_repo(svc_client, mock_redis): } }, { - 'url': '/datasets/create', + 'url': '/datasets.create', 'allowed_method': 'POST', 'headers': { 'Content-Type': 'application/json', @@ -640,7 +640,7 @@ def svc_client_with_repo(svc_client, mock_redis): } }, { - 'url': '/datasets/files-list', + 'url': '/datasets.files_list', 'allowed_method': 'GET', 'headers': { 'Content-Type': 'application/json', @@ -648,7 +648,7 @@ def svc_client_with_repo(svc_client, mock_redis): } }, { - 'url': '/datasets/list', + 'url': '/datasets.list', 'allowed_method': 'GET', 'headers': { 'Content-Type': 'application/json', diff --git a/renku/service/views/cache.py b/renku/service/views/cache.py index d0686dceaa..34c2b19074 100644 --- a/renku/service/views/cache.py +++ b/renku/service/views/cache.py @@ -46,7 +46,7 @@ @marshal_with(FileListResponseRPC) @header_doc(description='List uploaded files.', tags=(CACHE_BLUEPRINT_TAG, )) @cache_blueprint.route( - '/cache/files-list', + '/cache.files_list', methods=['GET'], provide_automatic_options=False, ) @@ -75,7 +75,7 @@ def list_uploaded_files_view(user, cache): tags=(CACHE_BLUEPRINT_TAG, ), ) @cache_blueprint.route( - '/cache/files-upload', + '/cache.files_upload', methods=['POST'], provide_automatic_options=False, ) @@ -171,7 +171,7 @@ def upload_file_view(user, cache): tags=(CACHE_BLUEPRINT_TAG, ) ) @cache_blueprint.route( - '/cache/project-clone', + '/cache.project_clone', methods=['POST'], provide_automatic_options=False, ) @@ -223,7 +223,7 @@ def project_clone(user, cache): tags=(CACHE_BLUEPRINT_TAG, ), ) @cache_blueprint.route( - '/cache/project-list', + '/cache.project_list', methods=['GET'], provide_automatic_options=False, ) diff --git a/renku/service/views/datasets.py b/renku/service/views/datasets.py index 2fb71f744f..fcbc8042ed 100644 --- a/renku/service/views/datasets.py +++ b/renku/service/views/datasets.py @@ -46,7 +46,7 @@ @marshal_with(DatasetListResponseRPC) @header_doc('List all datasets in project.', tags=(DATASET_BLUEPRINT_TAG, )) @dataset_blueprint.route( - '/datasets/list', + '/datasets.list', methods=['GET'], provide_automatic_options=False, ) @@ -85,7 +85,7 @@ def list_datasets_view(user, cache): @marshal_with(DatasetFilesListResponseRPC) @header_doc('List files in a dataset.', tags=(DATASET_BLUEPRINT_TAG, )) @dataset_blueprint.route( - '/datasets/files-list', + '/datasets.files_list', methods=['GET'], provide_automatic_options=False, ) @@ -129,7 +129,7 @@ def list_dataset_files_view(user, cache): 'Add uploaded file to cloned repository.', tags=(DATASET_BLUEPRINT_TAG, ) ) @dataset_blueprint.route( - '/datasets/add', + '/datasets.add', methods=['POST'], provide_automatic_options=False, ) @@ -192,7 +192,7 @@ def add_file_to_dataset_view(user, cache): 'Create a new dataset in a project.', tags=(DATASET_BLUEPRINT_TAG, ) ) @dataset_blueprint.route( - '/datasets/create', + '/datasets.create', methods=['POST'], provide_automatic_options=False, ) diff --git a/tests/service/test_cache_views.py b/tests/service/test_cache_views.py index 55e972e5b3..7735758aad 100644 --- a/tests/service/test_cache_views.py +++ b/tests/service/test_cache_views.py @@ -52,7 +52,7 @@ def test_list_upload_files_all(svc_client): 'accept': 'application/json', 'Renku-User-Id': 'user' } - response = svc_client.get('/cache/files-list', headers=headers_user) + response = svc_client.get('/cache.files_list', headers=headers_user) assert {'result'} == set(response.json.keys()) @@ -68,7 +68,7 @@ def test_list_upload_files_all_no_auth(svc_client): 'accept': 'application/json', } response = svc_client.get( - '/cache/files-list', + '/cache.files_list', headers=headers, ) @@ -84,7 +84,7 @@ def test_file_upload(svc_client): headers_user = {'Renku-User-Id': '{0}'.format(uuid.uuid4().hex)} response = svc_client.post( - '/cache/files-upload', + '/cache.files_upload', data=dict(file=(io.BytesIO(b'this is a test'), 'datafile.txt'), ), headers=headers_user, ) @@ -104,7 +104,7 @@ def test_file_upload_override(svc_client): headers_user = {'Renku-User-Id': '{0}'.format(uuid.uuid4().hex)} response = svc_client.post( - '/cache/files-upload', + '/cache.files_upload', data=dict(file=(io.BytesIO(b'this is a test'), 'datafile.txt'), ), headers=headers_user, ) @@ -119,7 +119,7 @@ def test_file_upload_override(svc_client): old_file_id = response.json['result']['files'][0]['file_id'] response = svc_client.post( - '/cache/files-upload', + '/cache.files_upload', data=dict(file=(io.BytesIO(b'this is a test'), 'datafile.txt'), ), headers=headers_user, ) @@ -132,7 +132,7 @@ def test_file_upload_override(svc_client): assert 'file exists' == response.json['error']['reason'] response = svc_client.post( - '/cache/files-upload', + '/cache.files_upload', data=dict(file=(io.BytesIO(b'this is a test'), 'datafile.txt'), ), query_string={'override_existing': True}, headers=headers_user, @@ -153,7 +153,7 @@ def test_file_upload_same_file(svc_client): """Check successful file upload with same file uploaded twice.""" headers_user1 = {'Renku-User-Id': '{0}'.format(uuid.uuid4().hex)} response = svc_client.post( - '/cache/files-upload', + '/cache.files_upload', data=dict(file=(io.BytesIO(b'this is a test'), 'datafile.txt'), ), headers=headers_user1, ) @@ -168,7 +168,7 @@ def test_file_upload_same_file(svc_client): ) response = svc_client.post( - '/cache/files-upload', + '/cache.files_upload', data=dict(file=(io.BytesIO(b'this is a test'), 'datafile.txt'), ), headers=headers_user1, ) @@ -184,7 +184,7 @@ def test_file_upload_same_file(svc_client): def test_file_upload_no_auth(svc_client): """Check failed file upload.""" response = svc_client.post( - '/cache/files-upload', + '/cache.files_upload', data=dict(file=(io.BytesIO(b'this is a test'), 'datafile.txt'), ), ) @@ -202,7 +202,7 @@ def test_file_upload_with_users(svc_client): headers_user2 = {'Renku-User-Id': '{0}'.format(uuid.uuid4().hex)} response = svc_client.post( - '/cache/files-upload', + '/cache.files_upload', data=dict(file=(io.BytesIO(b'this is a test'), 'datafile1.txt'), ), headers=headers_user1 ) @@ -214,7 +214,7 @@ def test_file_upload_with_users(svc_client): assert 200 == response.status_code response = svc_client.post( - '/cache/files-upload', + '/cache.files_upload', data=dict(file=(io.BytesIO(b'this is a test'), 'datafile1.txt'), ), headers=headers_user2 ) @@ -222,7 +222,7 @@ def test_file_upload_with_users(svc_client): assert response assert {'result'} == set(response.json.keys()) - response = svc_client.get('/cache/files-list', headers=headers_user1) + response = svc_client.get('/cache.files_list', headers=headers_user1) assert response @@ -243,7 +243,7 @@ def test_clone_projects_no_auth(svc_client): } response = svc_client.post( - '/cache/project-clone', data=json.dumps(payload) + '/cache.project_clone', data=json.dumps(payload) ) assert {'error'} == set(response.json.keys()) @@ -262,7 +262,7 @@ def test_clone_projects_no_auth(svc_client): } response = svc_client.post( - '/cache/project-clone', data=json.dumps(payload), headers=headers + '/cache.project_clone', data=json.dumps(payload), headers=headers ) assert response @@ -287,7 +287,7 @@ def test_clone_projects_with_auth(svc_client): } response = svc_client.post( - '/cache/project-clone', data=json.dumps(payload), headers=headers + '/cache.project_clone', data=json.dumps(payload), headers=headers ) assert response @@ -313,7 +313,7 @@ def test_clone_projects_multiple(svc_client): } response = svc_client.post( - '/cache/project-clone', data=json.dumps(payload), headers=headers + '/cache.project_clone', data=json.dumps(payload), headers=headers ) assert response @@ -322,7 +322,7 @@ def test_clone_projects_multiple(svc_client): project_ids.append(response.json['result']) response = svc_client.post( - '/cache/project-clone', data=json.dumps(payload), headers=headers + '/cache.project_clone', data=json.dumps(payload), headers=headers ) assert response @@ -330,7 +330,7 @@ def test_clone_projects_multiple(svc_client): project_ids.append(response.json['result']) response = svc_client.post( - '/cache/project-clone', data=json.dumps(payload), headers=headers + '/cache.project_clone', data=json.dumps(payload), headers=headers ) assert response @@ -338,14 +338,14 @@ def test_clone_projects_multiple(svc_client): project_ids.append(response.json['result']) response = svc_client.post( - '/cache/project-clone', data=json.dumps(payload), headers=headers + '/cache.project_clone', data=json.dumps(payload), headers=headers ) assert response assert {'result'} == set(response.json.keys()) last_pid = response.json['result']['project_id'] - response = svc_client.get('/cache/project-list', headers=headers) + response = svc_client.get('/cache.project_list', headers=headers) assert response assert {'result'} == set(response.json.keys()) @@ -374,7 +374,7 @@ def test_clone_projects_list_view_errors(svc_client): } response = svc_client.post( - '/cache/project-clone', data=json.dumps(payload), headers=headers + '/cache.project_clone', data=json.dumps(payload), headers=headers ) assert response assert {'result'} == set(response.json.keys()) @@ -384,7 +384,7 @@ def test_clone_projects_list_view_errors(svc_client): ) response = svc_client.get( - '/cache/project-list', + '/cache.project_list', # no auth headers, expected error ) assert response @@ -392,7 +392,7 @@ def test_clone_projects_list_view_errors(svc_client): assert {'error'} == set(response.json.keys()) assert INVALID_HEADERS_ERROR_CODE == response.json['error']['code'] - response = svc_client.get('/cache/project-list', headers=headers) + response = svc_client.get('/cache.project_list', headers=headers) assert response assert {'result'} == set(response.json.keys()) assert 1 == len(response.json['result']['projects']) @@ -420,7 +420,7 @@ def test_clone_projects_invalid_headers(svc_client): } response = svc_client.post( - '/cache/project-clone', + '/cache.project_clone', data=json.dumps(payload), headers=headers, ) @@ -428,7 +428,7 @@ def test_clone_projects_invalid_headers(svc_client): assert {'result'} == set(response.json.keys()) response = svc_client.get( - '/cache/project-list', + '/cache.project_list', # no auth headers, expected error ) @@ -436,7 +436,7 @@ def test_clone_projects_invalid_headers(svc_client): assert {'error'} == set(response.json.keys()) assert INVALID_HEADERS_ERROR_CODE == response.json['error']['code'] - response = svc_client.get('/cache/project-list', headers=headers) + response = svc_client.get('/cache.project_list', headers=headers) assert response assert {'result'} == set(response.json.keys()) assert 1 == len(response.json['result']['projects']) @@ -449,7 +449,7 @@ def test_upload_zip_unpack_archive(datapack_zip, svc_client_with_repo): headers.pop('Content-Type') response = svc_client.post( - '/cache/files-upload', + '/cache.files_upload', data=dict( file=(io.BytesIO(datapack_zip.read_bytes()), datapack_zip.name), ), @@ -478,7 +478,7 @@ def test_upload_zip_archive(datapack_zip, svc_client_with_repo): headers.pop('Content-Type') response = svc_client.post( - '/cache/files-upload', + '/cache.files_upload', data=dict( file=(io.BytesIO(datapack_zip.read_bytes()), datapack_zip.name), ), @@ -507,7 +507,7 @@ def test_upload_tar_unpack_archive(datapack_tar, svc_client_with_repo): headers.pop('Content-Type') response = svc_client.post( - '/cache/files-upload', + '/cache.files_upload', data=dict( file=(io.BytesIO(datapack_tar.read_bytes()), datapack_tar.name), ), @@ -536,7 +536,7 @@ def test_upload_tar_archive(datapack_tar, svc_client_with_repo): headers.pop('Content-Type') response = svc_client.post( - '/cache/files-upload', + '/cache.files_upload', data=dict( file=(io.BytesIO(datapack_tar.read_bytes()), datapack_tar.name), ), @@ -565,7 +565,7 @@ def test_field_upload_resp_fields(datapack_tar, svc_client_with_repo): headers.pop('Content-Type') response = svc_client.post( - '/cache/files-upload', + '/cache.files_upload', data=dict( file=(io.BytesIO(datapack_tar.read_bytes()), datapack_tar.name), ), diff --git a/tests/service/test_dataset_views.py b/tests/service/test_dataset_views.py index 6a8fb98245..fec3d6df18 100644 --- a/tests/service/test_dataset_views.py +++ b/tests/service/test_dataset_views.py @@ -38,7 +38,7 @@ def test_create_dataset_view(svc_client_with_repo): } response = svc_client.post( - '/datasets/create', + '/datasets.create', data=json.dumps(payload), headers=headers, ) @@ -62,7 +62,7 @@ def test_create_dataset_view_dataset_exists(svc_client_with_repo): } response = svc_client.post( - '/datasets/create', + '/datasets.create', data=json.dumps(payload), headers=headers, ) @@ -87,7 +87,7 @@ def test_create_dataset_view_unknown_param(svc_client_with_repo): } response = svc_client.post( - '/datasets/create', + '/datasets.create', data=json.dumps(payload), headers=headers, ) @@ -112,7 +112,7 @@ def test_create_dataset_with_no_identity(svc_client_with_repo): } response = svc_client.post( - '/datasets/create', + '/datasets.create', data=json.dumps(payload), headers={'Content-Type': headers['Content-Type']} # no user identity, expect error @@ -139,7 +139,7 @@ def test_add_file_view_with_no_identity(svc_client_with_repo): } response = svc_client.post( - '/datasets/add', + '/datasets.add', data=json.dumps(payload), headers={'Content-Type': headers['Content-Type']} # no user identity, expect error @@ -161,7 +161,7 @@ def test_add_file_view(svc_client_with_repo): content_type = headers.pop('Content-Type') response = svc_client.post( - '/cache/files-upload', + '/cache.files_upload', data=dict(file=(io.BytesIO(b'this is a test'), 'datafile1.txt'), ), query_string={'override_existing': True}, headers=headers @@ -186,7 +186,7 @@ def test_add_file_view(svc_client_with_repo): headers['Content-Type'] = content_type response = svc_client.post( - '/datasets/add', + '/datasets.add', data=json.dumps(payload), headers=headers, ) @@ -212,7 +212,7 @@ def test_list_datasets_view(svc_client_with_repo): } response = svc_client.get( - '/datasets/list', + '/datasets.list', query_string=params, headers=headers, ) @@ -237,7 +237,7 @@ def test_list_datasets_view_no_auth(svc_client_with_repo): } response = svc_client.get( - '/datasets/list', + '/datasets.list', query_string=params, ) @@ -257,7 +257,7 @@ def test_create_and_list_datasets_view(svc_client_with_repo): } response = svc_client.post( - '/datasets/create', + '/datasets.create', data=json.dumps(payload), headers=headers, ) @@ -273,7 +273,7 @@ def test_create_and_list_datasets_view(svc_client_with_repo): } response = svc_client.get( - '/datasets/list', + '/datasets.list', query_string=params_list, headers=headers, ) @@ -300,7 +300,7 @@ def test_list_dataset_files(svc_client_with_repo): file_name = '{0}'.format(uuid.uuid4().hex) response = svc_client.post( - '/cache/files-upload', + '/cache.files_upload', data=dict(file=(io.BytesIO(b'this is a test'), file_name), ), query_string={'override_existing': True}, headers=headers @@ -324,7 +324,7 @@ def test_list_dataset_files(svc_client_with_repo): headers['Content-Type'] = content_type response = svc_client.post( - '/datasets/add', + '/datasets.add', data=json.dumps(payload), headers=headers, ) @@ -342,7 +342,7 @@ def test_list_dataset_files(svc_client_with_repo): } response = svc_client.get( - '/datasets/files-list', + '/datasets.files_list', query_string=params, headers=headers, ) @@ -366,7 +366,7 @@ def test_add_with_unpacked_archive(datapack_zip, svc_client_with_repo): content_type = headers.pop('Content-Type') response = svc_client.post( - '/cache/files-upload', + '/cache.files_upload', data=dict( file=(io.BytesIO(datapack_zip.read_bytes()), datapack_zip.name), ), @@ -398,7 +398,7 @@ def test_add_with_unpacked_archive(datapack_zip, svc_client_with_repo): headers['Content-Type'] = content_type response = svc_client.post( - '/datasets/create', + '/datasets.create', data=json.dumps(payload), headers=headers, ) @@ -418,7 +418,7 @@ def test_add_with_unpacked_archive(datapack_zip, svc_client_with_repo): } response = svc_client.post( - '/datasets/add', + '/datasets.add', data=json.dumps(payload), headers=headers, ) @@ -436,7 +436,7 @@ def test_add_with_unpacked_archive(datapack_zip, svc_client_with_repo): } response = svc_client.get( - '/datasets/files-list', + '/datasets.files_list', query_string=params, headers=headers, ) @@ -460,7 +460,7 @@ def test_add_with_unpacked_archive_all(datapack_zip, svc_client_with_repo): content_type = headers.pop('Content-Type') response = svc_client.post( - '/cache/files-upload', + '/cache.files_upload', data=dict( file=(io.BytesIO(datapack_zip.read_bytes()), datapack_zip.name), ), @@ -495,7 +495,7 @@ def test_add_with_unpacked_archive_all(datapack_zip, svc_client_with_repo): headers['Content-Type'] = content_type response = svc_client.post( - '/datasets/create', + '/datasets.create', data=json.dumps(payload), headers=headers, ) @@ -513,7 +513,7 @@ def test_add_with_unpacked_archive_all(datapack_zip, svc_client_with_repo): } response = svc_client.post( - '/datasets/add', + '/datasets.add', data=json.dumps(payload), headers=headers, ) @@ -531,7 +531,7 @@ def test_add_with_unpacked_archive_all(datapack_zip, svc_client_with_repo): } response = svc_client.get( - '/datasets/files-list', + '/datasets.files_list', query_string=params, headers=headers, ) From 459199e8b28b6cee94581f15dee7a40b3171a2ac Mon Sep 17 00:00:00 2001 From: sam Date: Mon, 16 Dec 2019 16:44:44 +0100 Subject: [PATCH 08/11] feat: set custom commit message --- renku/service/serializers/datasets.py | 26 ++++++++++- renku/service/views/datasets.py | 17 ++++++- tests/service/test_dataset_views.py | 67 +++++++++++++++++++++++++++ 3 files changed, 106 insertions(+), 4 deletions(-) diff --git a/renku/service/serializers/datasets.py b/renku/service/serializers/datasets.py index 154a0a8623..b7a86c783a 100644 --- a/renku/service/serializers/datasets.py +++ b/renku/service/serializers/datasets.py @@ -16,7 +16,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """Renku service datasets serializers.""" -from marshmallow import Schema, fields +from marshmallow import Schema, fields, pre_load, post_load from renku.service.serializers.rpc import JsonRPCResponse @@ -31,11 +31,22 @@ class DatasetAuthors(Schema): class DatasetCreateRequest(Schema): """Request schema for dataset create view.""" + authors = fields.List(fields.Nested(DatasetAuthors)) + commit_message = fields.String() dataset_name = fields.String(required=True) description = fields.String() - authors = fields.List(fields.Nested(DatasetAuthors)) project_id = fields.String(required=True) + @pre_load() + def default_commit_message(self, data, **kwargs): + """Set default commit message.""" + if not data.get('commit_message'): + data['commit_message'] = 'service: dataset create {0}'.format( + data['dataset_name'] + ) + + return data + class DatasetCreateResponse(Schema): """Response schema for dataset create view.""" @@ -58,11 +69,22 @@ class DatasetAddFile(Schema): class DatasetAddRequest(Schema): """Request schema for dataset add file view.""" + commit_message = fields.String() dataset_name = fields.String(required=True) create_dataset = fields.Boolean(missing=False) project_id = fields.String(required=True) files = fields.List(fields.Nested(DatasetAddFile), required=True) + @post_load() + def default_commit_message(self, data, **kwargs): + """Set default commit message.""" + if not data.get('commit_message'): + data['commit_message'] = 'service: dataset add {0}'.format( + data['dataset_name'] + ) + + return data + class DatasetAddResponse(Schema): """Response schema for dataset add file view.""" diff --git a/renku/service/views/datasets.py b/renku/service/views/datasets.py index fcbc8042ed..38c9e5c3d1 100644 --- a/renku/service/views/datasets.py +++ b/renku/service/views/datasets.py @@ -145,6 +145,7 @@ def add_file_to_dataset_view(user, cache): ctx = DatasetAddRequest().load(request.json) project = cache.get_project(user, ctx['project_id']) project_path = make_project_path(user, project) + if not project_path: return jsonify( error={ @@ -153,6 +154,11 @@ def add_file_to_dataset_view(user, cache): } ) + if not ctx['commit_message']: + ctx['commit_message'] = 'service: dataset add {0}'.format( + ctx['dataset_name'] + ) + local_paths = [] for file_ in ctx['files']: file = cache.get_file(user, file_['file_id']) @@ -164,11 +170,16 @@ def add_file_to_dataset_view(user, cache): 'message': 'invalid file_id: {0}'.format(file_['file_id']) } ) + + ctx['commit_message'] += ' {0}'.format(local_path.name) local_paths.append(str(local_path)) with chdir(project_path): add_file( - local_paths, ctx['dataset_name'], create=ctx['create_dataset'] + local_paths, + ctx['dataset_name'], + create=ctx['create_dataset'], + commit_message=ctx['commit_message'] ) if not repo_sync(project_path): @@ -218,7 +229,9 @@ def create_dataset_view(user, cache): ) with chdir(project_path): - create_dataset(ctx['dataset_name']) + create_dataset( + ctx['dataset_name'], commit_message=ctx['commit_message'] + ) if not repo_sync(project_path): return jsonify( diff --git a/tests/service/test_dataset_views.py b/tests/service/test_dataset_views.py index fec3d6df18..1864b7bfec 100644 --- a/tests/service/test_dataset_views.py +++ b/tests/service/test_dataset_views.py @@ -50,6 +50,31 @@ def test_create_dataset_view(svc_client_with_repo): assert payload['dataset_name'] == response.json['result']['dataset_name'] +@pytest.mark.service +@pytest.mark.integration +def test_create_dataset_commit_msg(svc_client_with_repo): + """Create new dataset successfully with custom commit message.""" + svc_client, headers, project_id = svc_client_with_repo + + payload = { + 'project_id': project_id, + 'dataset_name': '{0}'.format(uuid.uuid4().hex), + 'commit_message': 'my awesome dataset' + } + + response = svc_client.post( + '/datasets.create', + data=json.dumps(payload), + headers=headers, + ) + + assert response + + assert {'result'} == set(response.json.keys()) + assert {'dataset_name'} == set(response.json['result'].keys()) + assert payload['dataset_name'] == response.json['result']['dataset_name'] + + @pytest.mark.service @pytest.mark.integration def test_create_dataset_view_dataset_exists(svc_client_with_repo): @@ -201,6 +226,48 @@ def test_add_file_view(svc_client_with_repo): assert file_id == response.json['result']['files'][0]['file_id'] +@pytest.mark.service +@pytest.mark.integration +def test_add_file_commit_msg(svc_client_with_repo): + """Check adding of uploaded file to dataset with custom commit message.""" + svc_client, headers, project_id = svc_client_with_repo + content_type = headers.pop('Content-Type') + + response = svc_client.post( + '/cache.files_upload', + data=dict(file=(io.BytesIO(b'this is a test'), 'datafile1.txt'), ), + query_string={'override_existing': True}, + headers=headers + ) + + file_id = response.json['result']['files'][0]['file_id'] + assert isinstance(uuid.UUID(file_id), uuid.UUID) + + payload = { + 'commit_message': 'my awesome data file', + 'project_id': project_id, + 'dataset_name': '{0}'.format(uuid.uuid4().hex), + 'create_dataset': True, + 'files': [{ + 'file_id': file_id, + }, ] + } + headers['Content-Type'] = content_type + response = svc_client.post( + '/datasets.add', + data=json.dumps(payload), + headers=headers, + ) + + assert response + assert {'result'} == set(response.json.keys()) + assert {'dataset_name', 'project_id', + 'files'} == set(response.json['result'].keys()) + + assert 1 == len(response.json['result']['files']) + assert file_id == response.json['result']['files'][0]['file_id'] + + @pytest.mark.service @pytest.mark.integration def test_list_datasets_view(svc_client_with_repo): From 3496769084bde861de8d1e6027ff3dfec8838706 Mon Sep 17 00:00:00 2001 From: sam Date: Mon, 16 Dec 2019 17:18:01 +0100 Subject: [PATCH 09/11] chore: remove defensive scoping of git options --- renku/core/management/clone.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/renku/core/management/clone.py b/renku/core/management/clone.py index 11a8274598..9c7a49f4a4 100644 --- a/renku/core/management/clone.py +++ b/renku/core/management/clone.py @@ -68,11 +68,7 @@ def clone( for key, value in config.items(): key_path = key.split('.') - if len(key_path) != 2: - raise errors.GitError( - 'Cannot write to config path: {0}'.format(key) - ) - config_writer.set_value(key_path[0], key_path[1], value) + config_writer.set_value(key_path[0], '.'.join(key_path[1:]), value) config_writer.release() From 7b214d64e5299ef6bcf524b021e84d834da64b01 Mon Sep 17 00:00:00 2001 From: sam Date: Tue, 17 Dec 2019 12:30:42 +0100 Subject: [PATCH 10/11] chore: added IndexError exception sentinel --- renku/core/management/clone.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/renku/core/management/clone.py b/renku/core/management/clone.py index 9c7a49f4a4..fa9ca82004 100644 --- a/renku/core/management/clone.py +++ b/renku/core/management/clone.py @@ -68,7 +68,14 @@ def clone( for key, value in config.items(): key_path = key.split('.') - config_writer.set_value(key_path[0], '.'.join(key_path[1:]), value) + key = key_path.pop() + + if not key_path or not key: + raise errors.GitError( + 'Cannot write to config. Section path or key is invalid.' + ) + + config_writer.set_value('.'.join(key_path), key, value) config_writer.release() From 6cad789201d6de98480103ded16595d69dc08c5c Mon Sep 17 00:00:00 2001 From: sam Date: Tue, 17 Dec 2019 15:52:24 +0100 Subject: [PATCH 11/11] chore: added test for renku clone with custom config --- renku/core/commands/clone.py | 1 - renku/core/management/clone.py | 2 +- tests/cli/test_integration_datasets.py | 25 +++++++++++++++++++++++++ 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/renku/core/commands/clone.py b/renku/core/commands/clone.py index c2cd6c9a47..fe435b89d6 100644 --- a/renku/core/commands/clone.py +++ b/renku/core/commands/clone.py @@ -34,7 +34,6 @@ def renku_clone( progress=None, config=None, raise_git_except=False, - commit_message=None ): """Clone Renku project repo, install Git hooks and LFS.""" install_lfs = client.use_external_storage diff --git a/renku/core/management/clone.py b/renku/core/management/clone.py index fa9ca82004..41120bb04c 100644 --- a/renku/core/management/clone.py +++ b/renku/core/management/clone.py @@ -74,7 +74,7 @@ def clone( raise errors.GitError( 'Cannot write to config. Section path or key is invalid.' ) - + config_writer.set_value('.'.join(key_path), key, value) config_writer.release() diff --git a/tests/cli/test_integration_datasets.py b/tests/cli/test_integration_datasets.py index 06f94c8b29..6604ca7c8b 100644 --- a/tests/cli/test_integration_datasets.py +++ b/tests/cli/test_integration_datasets.py @@ -24,6 +24,8 @@ import pytest from renku.cli import cli +from renku.core.commands.clone import renku_clone +from renku.core.utils.contexts import chdir @pytest.mark.parametrize( @@ -929,6 +931,29 @@ def test_renku_clone(runner, monkeypatch): assert 1 == result.exit_code +@pytest.mark.integration +def test_renku_clone_with_config(tmpdir): + """Test cloning of a Renku repo and existence of required settings.""" + REMOTE = 'https://dev.renku.ch/gitlab/virginiafriedrich/datasets-test.git' + + with chdir(tmpdir): + renku_clone( + REMOTE, + config={ + 'user.name': 'sam', + 'user.email': 's@m.i', + 'filter.lfs.custom': '0' + } + ) + + repo = git.Repo('datasets-test') + reader = repo.config_reader() + reader.values() + + lfs_config = dict(reader.items('filter.lfs')) + assert '0' == lfs_config.get('custom') + + @pytest.mark.integration @pytest.mark.parametrize( 'path,expected_path', [('', 'datasets-test'), ('.', '.'),