Merge branch 'feature/backend_couchdb' into 'master'

Feature/backend couchdb

Closes #88

See merge request !50

.gitlab-ci.yml

@@ -33,7 +33,7 @@ test:
   - pip install --cache-dir="$PIP_CACHE_DIR" unittest-xml-reporting coverage
   - pip install --cache-dir="$PIP_CACHE_DIR" -r requirements.txt
   # Setup test config and CouchDB database server
-  - echo -e "[couchdb]\nurl = http://couchdb:5984" > test/test_config.ini
+  - echo -e "[couchdb]\nurl = http://couchdb:5984\n" > test/test_config.ini
   - python test/_helper/setup_testdb.py -u "$COUCHDB_USER" -p "$COUCHDB_PASSWORD"
   # Add source directory to PYTHONPATH to allow testing our CLI scripts, which import our modules
   - export PYTHONPATH=".:$PYTHONPATH"

aas/model/base.py

@@ -22,7 +22,7 @@ from typing import List, Optional, Set, TypeVar, MutableSet, Generic, Iterable,
 import re
 
 from . import datatypes
-from .. import backends
+from ..backend import backends
 
 if TYPE_CHECKING:
     from . import provider
@@ -541,7 +541,7 @@ class Referable(metaclass=abc.ABCMeta):
             break
         return None, None
 
-    def update_from(self, other: "Referable"):
+    def update_from(self, other: "Referable", update_source: bool = False):
         """
         Internal function to updates the object's attributes from another object of a similar type.
 
@@ -549,9 +549,12 @@ class Referable(metaclass=abc.ABCMeta):
         protocol clients, etc.) to update the object's data, after `update()` has been called.
 
         :param other: The object to update from
+        :param update_source: Update the source attribute with the other's source attribute. This is not propagated
+                              recursively
         """
         for name, var in vars(other).items():
-            if name == "parent":  # do not update the parent
+            # do not update the parent or source (depending on update_source parameter)
+            if name == "parent" or name == "source" and not update_source:
                 continue
             if isinstance(var, NamespaceSet):
                 # update the elements of the NameSpaceSet
@@ -1126,7 +1129,7 @@ class NamespaceSet(MutableSet[_RT], Generic[_RT]):
                 referable = self._backend[other_referable.id_short]
                 if type(referable) is type(other_referable):
                     # referable is the same as other referable
-                    referable.update_from(other_referable)
+                    referable.update_from(other_referable, update_source=True)
             except KeyError:
                 # other referable is not in NamespaceSet
                 referables_to_add.append(other_referable)

test/test_backends.py → test/backend/test_backends.py

 from unittest import mock
 import unittest
 
-from aas import backends
+from aas.backend import backends
 
 
 class BackendsTest(unittest.TestCase):
     def test_backend_store(self):
-        with mock.patch("aas.backends.Backend") as mock_backend:
+        with mock.patch("aas.backend.backends.Backend") as mock_backend:
             backends.register_backend("mockScheme", mock_backend)
             self.assertEqual(backends.get_backend("mockScheme:x-test:test_backend"), mock_backend)
 

test/adapter/test_couchdb.py → test/backend/test_couchdb.py

@@ -9,15 +9,15 @@
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
 # specific language governing permissions and limitations under the License.
 import base64
-import concurrent.futures
 import configparser
 import copy
 import os
 import unittest
+import unittest.mock
 import urllib.request
 import urllib.error
 
-from aas.adapter import couchdb
+from aas.backend import backends, couchdb
 from aas.examples.data.example_aas import *
 
 
@@ -25,8 +25,11 @@ TEST_CONFIG = configparser.ConfigParser()
 TEST_CONFIG.read((os.path.join(os.path.dirname(__file__), "..", "test_config.default.ini"),
                   os.path.join(os.path.dirname(__file__), "..", "test_config.ini")))
 
+source_core: str = "couchdb://" + TEST_CONFIG["couchdb"]["url"].lstrip("http://") + "/" + \
+                   TEST_CONFIG["couchdb"]["database"] + "/"
 
-# Check if CouchDB database is avalable. Otherwise, skip tests.
+
+# Check if CouchDB database is available. Otherwise, skip tests.
 try:
     request = urllib.request.Request(
         "{}/{}".format(TEST_CONFIG['couchdb']['url'], TEST_CONFIG['couchdb']['database']),
@@ -44,173 +47,153 @@ except urllib.error.URLError as e:
     COUCHDB_ERROR = e
 
 
+class CouchDBBackendOfflineMethodsTest(unittest.TestCase):
+    def test_parse_source(self):
+        couchdb.register_credentials(url="couchdb.plt.rwth-aachen.de:5984",
+                                     username="test_user",
+                                     password="test_password")
+
+        url = couchdb.CouchDBBackend._parse_source(
+            "couchdbs://couchdb.plt.rwth-aachen.de:5984/path_to_db/path_to_doc"
+        )
+        expected_url = "https://couchdb.plt.rwth-aachen.de:5984/path_to_db/path_to_doc"
+        self.assertEqual(expected_url, url)
+
+        url = couchdb.CouchDBBackend._parse_source(
+            "couchdb://couchdb.plt.rwth-aachen.de:5984/path_to_db/path_to_doc"
+        )
+        expected_url = "http://couchdb.plt.rwth-aachen.de:5984/path_to_db/path_to_doc"
+        self.assertEqual(expected_url, url)
+
+        with self.assertRaises(couchdb.CouchDBSourceError) as cm:
+            couchdb.CouchDBBackend._parse_source("wrong_scheme:plt.rwth-aachen.couchdb:5984/path_to_db/path_to_doc")
+            self.assertEqual("Source has wrong format. "
+                             "Expected to start with {couchdb, couchdbs}, got "
+                             "{wrong_scheme:plt.rwth-aachen.couchdb:5984/path_to_db/path_to_doc}",
+                             cm.exception)
+
+
 @unittest.skipUnless(COUCHDB_OKAY, "No CouchDB is reachable at {}/{}: {}".format(TEST_CONFIG['couchdb']['url'],
                                                                                  TEST_CONFIG['couchdb']['database'],
                                                                                  COUCHDB_ERROR))
-class CouchDBTest(unittest.TestCase):
+class CouchDBBackendTest(unittest.TestCase):
     def setUp(self) -> None:
-        # Create CouchDB store, login and check database
-        self.db = couchdb.CouchDBObjectStore(TEST_CONFIG['couchdb']['url'], TEST_CONFIG['couchdb']['database'])
-        self.db.login(TEST_CONFIG['couchdb']['user'], TEST_CONFIG['couchdb']['password'])
-        self.db.check_database()
+        self.object_store = couchdb.CouchDBObjectStore(TEST_CONFIG['couchdb']['url'],
+                                                       TEST_CONFIG['couchdb']['database'])
+        couchdb.register_credentials(TEST_CONFIG["couchdb"]["url"],
+                                     TEST_CONFIG["couchdb"]["user"],
+                                     TEST_CONFIG["couchdb"]["password"])
+        backends.register_backend("couchdb", couchdb.CouchDBBackend)
+        self.object_store.check_database()
 
     def tearDown(self) -> None:
-        self.db.clear()
-        self.db.logout()
+        self.object_store.clear()
+
+    def test_object_store_add(self):
+        test_object = create_example_submodel()
+        self.object_store.add(test_object)
+        self.assertEqual(test_object.source, source_core+"IRI-https%3A%2F%2Facplt.org%2FTest_Submodel")
 
     def test_example_submodel_storing(self) -> None:
         example_submodel = create_example_submodel()
 
         # Add exmaple submodel
-        self.db.add(example_submodel)
-        self.assertEqual(1, len(self.db))
-        self.assertIn(example_submodel, self.db)
+        self.object_store.add(example_submodel)
+        self.assertEqual(1, len(self.object_store))
+        self.assertIn(example_submodel, self.object_store)
 
         # Restore example submodel and check data
-        submodel_restored = self.db.get_identifiable(
+        submodel_restored = self.object_store.get_identifiable(
             model.Identifier(id_='https://acplt.org/Test_Submodel', id_type=model.IdentifierType.IRI))
         assert (isinstance(submodel_restored, model.Submodel))
         checker = AASDataChecker(raise_immediately=True)
         check_example_submodel(checker, submodel_restored)
 
         # Delete example submodel
-        self.db.discard(submodel_restored)
-        self.assertNotIn(example_submodel, self.db)
+        self.object_store.discard(submodel_restored)
+        self.assertNotIn(example_submodel, self.object_store)
 
     def test_iterating(self) -> None:
         example_data = create_full_example()
 
         # Add all objects
         for item in example_data:
-            self.db.add(item)
+            self.object_store.add(item)
 
-        self.assertEqual(6, len(self.db))
+        self.assertEqual(6, len(self.object_store))
 
         # Iterate objects, add them to a DictObjectStore and check them
         retrieved_data_store: model.provider.DictObjectStore[model.Identifiable] = model.provider.DictObjectStore()
-        for item in self.db:
-            retrieved_data_store.add(item)
-        checker = AASDataChecker(raise_immediately=True)
-        check_full_example(checker, retrieved_data_store)
-
-    def test_parallel_iterating(self) -> None:
-        example_data = create_full_example()
-        ids = [item.identification for item in example_data]
-
-        # Add objects via thread pool executor
-        with concurrent.futures.ThreadPoolExecutor() as pool:
-            result = pool.map(self.db.add, example_data)
-        list(result)  # Iterate Executor result to raise exceptions
-
-        self.assertEqual(6, len(self.db))
-
-        # Retrieve objects via thread pool executor
-        with concurrent.futures.ThreadPoolExecutor() as pool:
-            retrieved_objects = pool.map(self.db.get_identifiable, ids)
-
-        retrieved_data_store: model.provider.DictObjectStore[model.Identifiable] = model.provider.DictObjectStore()
-        for item in retrieved_objects:
+        for item in self.object_store:
             retrieved_data_store.add(item)
-        self.assertEqual(6, len(retrieved_data_store))
         checker = AASDataChecker(raise_immediately=True)
         check_full_example(checker, retrieved_data_store)
 
-        # Delete objects via thread pool executor
-        with concurrent.futures.ThreadPoolExecutor() as pool:
-            result = pool.map(self.db.discard, example_data)
-        list(result)  # Iterate Executor result to raise exceptions
-
-        self.assertEqual(0, len(self.db))
-
     def test_key_errors(self) -> None:
         # Double adding an object should raise a KeyError
         example_submodel = create_example_submodel()
-        self.db.add(example_submodel)
+        self.object_store.add(example_submodel)
         with self.assertRaises(KeyError) as cm:
-            self.db.add(example_submodel)
+            self.object_store.add(example_submodel)
         self.assertEqual("'Identifiable with id Identifier(IRI=https://acplt.org/Test_Submodel) already exists in "
                          "CouchDB database'", str(cm.exception))
 
         # Querying a deleted object should raise a KeyError
-        retrieved_submodel = self.db.get_identifiable(
+        retrieved_submodel = self.object_store.get_identifiable(
             model.Identifier('https://acplt.org/Test_Submodel', model.IdentifierType.IRI))
-        self.db.discard(example_submodel)
+        self.object_store.discard(example_submodel)
         with self.assertRaises(KeyError) as cm:
-            self.db.get_identifiable(model.Identifier('https://acplt.org/Test_Submodel', model.IdentifierType.IRI))
+            self.object_store.get_identifiable(model.Identifier('https://acplt.org/Test_Submodel',
+                                                                model.IdentifierType.IRI))
         self.assertEqual("'No Identifiable with id IRI-https://acplt.org/Test_Submodel found in CouchDB database'",
                          str(cm.exception))
 
         # Double deleting should also raise a KeyError
         with self.assertRaises(KeyError) as cm:
-            self.db.discard(retrieved_submodel)
+            self.object_store.discard(retrieved_submodel)
         self.assertEqual("'No AAS object with id Identifier(IRI=https://acplt.org/Test_Submodel) exists in "
                          "CouchDB database'", str(cm.exception))
 
-    def test_conflict_errors(self) -> None:
+    def test_conflict_errors(self):
         # Preperation: add object and retrieve it from the database
         example_submodel = create_example_submodel()
-        self.db.add(example_submodel)
-        retrieved_submodel = self.db.get_identifiable(
+        self.object_store.add(example_submodel)
+        retrieved_submodel = self.object_store.get_identifiable(
             model.Identifier('https://acplt.org/Test_Submodel', model.IdentifierType.IRI))
 
-        # Simulate a concurrent modification
-        remote_modified_submodel = copy.copy(retrieved_submodel)
-        remote_modified_submodel.id_short = "newIdShort"
-        remote_modified_submodel.commit_changes()
+        # Simulate a concurrent modification (Commit submodel, while preventing that the couchdb revision store is
+        # updated)
+        with unittest.mock.patch("aas.backend.couchdb.set_couchdb_revision"):
+            retrieved_submodel.commit()
 
         # Committing changes to the retrieved object should now raise a conflict error
         retrieved_submodel.id_short = "myOtherNewIdShort"
         with self.assertRaises(couchdb.CouchDBConflictError) as cm:
-            retrieved_submodel.commit_changes()
+            retrieved_submodel.commit()
         self.assertEqual("Could not commit changes to id Identifier(IRI=https://acplt.org/Test_Submodel) due to a "
                          "concurrent modification in the database.", str(cm.exception))
 
         # Deleting the submodel with safe_delete should also raise a conflict error. Deletion without safe_delete should
         # work
         with self.assertRaises(couchdb.CouchDBConflictError) as cm:
-            self.db.discard(retrieved_submodel, True)
+            self.object_store.discard(retrieved_submodel, True)
         self.assertEqual("Object with id Identifier(IRI=https://acplt.org/Test_Submodel) has been modified in the "
                          "database since the version requested to be deleted.", str(cm.exception))
-        self.db.discard(retrieved_submodel, False)
-        self.assertEqual(0, len(self.db))
-
-        # Committing after deletion should also raise a conflict error
-        with self.assertRaises(couchdb.CouchDBConflictError) as cm:
-            retrieved_submodel.commit_changes()
-        self.assertEqual("Could not commit changes to id Identifier(IRI=https://acplt.org/Test_Submodel) due to a "
-                         "concurrent modification in the database.", str(cm.exception))
+        self.object_store.discard(retrieved_submodel, False)
+        self.assertEqual(0, len(self.object_store))
 
-    def test_editing(self) -> None:
-        example_submodel = create_example_submodel()
-        self.db.add(example_submodel)
+        # Committing after deletion should not raise a conflict error due to removal of the source attribute
+        retrieved_submodel.commit()
 
-        # Retrieve submodel from database and change ExampleCapability's semanticId
-        submodel = self.db.get_identifiable(
-            model.Identifier('https://acplt.org/Test_Submodel', model.IdentifierType.IRI))
-        assert(isinstance(submodel, couchdb.CouchDBSubmodel))
-        capability = submodel.submodel_element.get_referable('ExampleCapability')
-        capability.semantic_id = model.Reference((model.Key(type_=model.KeyElements.GLOBAL_REFERENCE,
-                                                  local=False,
-                                                  value='http://acplt.org/Capabilities/AnotherCapability',
-                                                  id_type=model.KeyType.IRDI),))
+    def test_editing(self):
+        test_object = create_example_submodel()
+        self.object_store.add(test_object)
 
-        # Commit changes
-        submodel.commit_changes()
+        # Test if commit uploads changes
+        test_object.id_short = "SomeNewIdShort"
+        test_object.commit()
 
-        # Change ExampleSubmodelCollectionOrdered's description
-        collection = submodel.submodel_element.get_referable('ExampleSubmodelCollectionOrdered')
-        collection.description['de'] = "Eine sehr wichtige Sammlung von Elementen"   # type: ignore
-
-        # Commit changes
-        submodel.commit_changes()
-
-        # Check version in database
-        new_submodel = self.db.get_identifiable(
-            model.Identifier('https://acplt.org/Test_Submodel', model.IdentifierType.IRI))
-        assert(isinstance(new_submodel, couchdb.CouchDBSubmodel))
-        capability = new_submodel.submodel_element.get_referable('ExampleCapability')
-        assert(isinstance(capability, model.Capability))
-        self.assertEqual('http://acplt.org/Capabilities/AnotherCapability',
-                         capability.semantic_id.key[0].value)  # type: ignore
-        collection = new_submodel.submodel_element.get_referable('ExampleSubmodelCollectionOrdered')
-        self.assertEqual("Eine sehr wichtige Sammlung von Elementen", collection.description['de'])  # type: ignore
+        # Test if update restores changes
+        test_object.id_short = "AnotherIdShort"
+        test_object.update()
+        self.assertEqual("SomeNewIdShort", test_object.id_short)

test/model/test_base.py

@@ -13,7 +13,8 @@ import unittest
 from unittest import mock
 from typing import Optional, List
 
-from aas import model, backends
+from aas import model
+from aas.backend import backends
 from aas.model import Identifier, Identifiable