[Feature] Package Export/Import (#116)

Fixes #90
Fixes #91
Fixes #115
Fixes #104

Co-authored-by: Tim Lorsbach <tim@lorsba.ch>
Reviewed-on: enviPath/enviPy#116

.gitignore

@@ -4,4 +4,5 @@ db.sqlite3
 static/admin/
 static/django_extensions/
 .env
-debug.log
+debug.log
+scratches/

epdb/logic.py

@@ -10,6 +10,7 @@ from django.conf import settings as s
 from epdb.models import User, Package, UserPackagePermission, GroupPackagePermission, Permission, Group, Setting, \
     EPModel, UserSettingPermission, Rule, Pathway, Node, Edge, Compound, Reaction, CompoundStructure
 from utilities.chem import FormatConverter
+from utilities.misc import PackageImporter, PackageExporter
 
 logger = logging.getLogger(__name__)
 
@@ -324,17 +325,6 @@ class PackageManager(object):
             return True
         return False
 
-    # @staticmethod
-    # def get_package_permission(user: 'User', package: Union[str, 'Package']):
-    #     if PackageManager.administrable(user, package):
-    #         return Permission.ALL[0]
-    #     elif PackageManager.writable(user, package):
-    #         return Permission.WRITE[0]
-    #     elif PackageManager.readable(user, package):
-    #         return Permission.READ[0]
-    #     else:
-    #         return None
-
     @staticmethod
     def has_package_permission(user: 'User', package: Union[str, 'Package'], permission: str):
 
@@ -491,7 +481,7 @@ class PackageManager(object):
 
     @staticmethod
     @transaction.atomic
-    def import_package(data: dict, owner: User, keep_ids=False, add_import_timestamp=True, trust_reviewed=False):
+    def import_legacy_package(data: dict, owner: User, keep_ids=False, add_import_timestamp=True, trust_reviewed=False):
         from uuid import UUID, uuid4
         from datetime import datetime
         from collections import defaultdict
@@ -872,6 +862,28 @@ class PackageManager(object):
 
         return pack
 
+    @staticmethod
+    @transaction.atomic
+    def import_pacakge(data: Dict[str, Any], owner: User, preserve_uuids=False, add_import_timestamp=True,
+                       trust_reviewed=False) -> Package:
+
+        importer = PackageImporter(data, preserve_uuids, add_import_timestamp, trust_reviewed)
+        imported_package = importer.do_import()
+
+        up = UserPackagePermission()
+        up.user = owner
+        up.package = imported_package
+        up.permission = up.ALL[0]
+        up.save()
+
+        return imported_package
+
+    @staticmethod
+    def export_package(package: Package, include_models: bool = False,
+                       include_external_identifiers: bool = True) -> Dict[str, Any]:
+        return PackageExporter(package).do_export()
+
+
 class SettingManager(object):
     setting_pattern = re.compile(r".*/setting/[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$")
 

epdb/management/commands/bootstrap.py

@@ -12,21 +12,24 @@ class Command(BaseCommand):
 
     def create_users(self):
 
-        if not User.objects.filter(email='anon@lorsba.ch').exists():
-            anon = UserManager.create_user("anonymous", "anon@lorsba.ch", "SuperSafe", is_active=True,
-                                           add_to_group=False, set_setting=False)
+        # Anonymous User
+        if not User.objects.filter(email='anon@envipath.com').exists():
+            anon = UserManager.create_user("anonymous", "anon@envipath.com", "SuperSafe",
+                                           is_active=True, add_to_group=False, set_setting=False)
         else:
-            anon = User.objects.get(email='anon@lorsba.ch')
+            anon = User.objects.get(email='anon@envipath.com')
 
-        if not User.objects.filter(email='admin@lorsba.ch').exists():
-            admin = UserManager.create_user("admin", "admin@lorsba.ch", "SuperSafe", is_active=True, add_to_group=False,
-                                            set_setting=False)
+        # Admin User
+        if not User.objects.filter(email='admin@envipath.com').exists():
+            admin = UserManager.create_user("admin", "admin@envipath.com", "SuperSafe",
+                                            is_active=True, add_to_group=False, set_setting=False)
             admin.is_staff = True
             admin.is_superuser = True
             admin.save()
         else:
-            admin = User.objects.get(email='admin@lorsba.ch')
+            admin = User.objects.get(email='admin@envipath.com')
 
+        # System Group
         g = GroupManager.create_group(admin, 'enviPath Users', 'All enviPath Users')
         g.public = True
         g.save()
@@ -40,25 +43,25 @@ class Command(BaseCommand):
         admin.default_group = g
         admin.save()
 
-        if not User.objects.filter(email='jebus@lorsba.ch').exists():
-            jebus = UserManager.create_user("jebus", "jebus@lorsba.ch", "SuperSafe", is_active=True, add_to_group=False,
-                                            set_setting=False)
-            jebus.is_staff = True
-            jebus.is_superuser = True
-            jebus.save()
+        if not User.objects.filter(email='user0@envipath.com').exists():
+            user0 = UserManager.create_user("user0", "user0@envipath.com", "SuperSafe",
+                                            is_active=True, add_to_group=False, set_setting=False)
+            user0.is_staff = True
+            user0.is_superuser = True
+            user0.save()
         else:
-            jebus = User.objects.get(email='jebus@lorsba.ch')
+            user0 = User.objects.get(email='user0@envipath.com')
 
-        g.user_member.add(jebus)
+        g.user_member.add(user0)
         g.save()
 
-        jebus.default_group = g
-        jebus.save()
+        user0.default_group = g
+        user0.save()
 
-        return anon, admin, g, jebus
+        return anon, admin, g, user0
 
     def import_package(self, data, owner):
-        return PackageManager.import_package(data, owner, keep_ids=True, add_import_timestamp=False, trust_reviewed=True)
+        return PackageManager.import_legacy_package(data, owner, keep_ids=True, add_import_timestamp=False, trust_reviewed=True)
 
     def create_default_setting(self, owner, packages):
         s = SettingManager.create_setting(
@@ -108,13 +111,6 @@ class Command(BaseCommand):
                 'base_url': 'https://www.rhea-db.org',
                 'url_pattern': 'https://www.rhea-db.org/rhea/{id}'
             },
-            {
-                'name': 'CAS',
-                'full_name': 'Chemical Abstracts Service Registry',
-                'description': 'Registry of chemical substances',
-                'base_url': 'https://www.cas.org',
-                'url_pattern': None  # CAS doesn't have a free public URL pattern
-            },
             {
                 'name': 'KEGG Reaction',
                 'full_name': 'KEGG Reaction Database',
@@ -122,13 +118,6 @@ class Command(BaseCommand):
                 'base_url': 'https://www.genome.jp',
                 'url_pattern': 'https://www.genome.jp/entry/reaction+{id}'
             },
-            {
-                'name': 'MetaCyc',
-                'full_name': 'MetaCyc Metabolic Pathway Database',
-                'description': 'Database of metabolic pathways and enzymes',
-                'base_url': 'https://metacyc.org',
-                'url_pattern': None
-            },
             {
                 'name': 'UniProt',
                 'full_name': 'MetaCyc Metabolic Pathway Database',
@@ -147,7 +136,9 @@ class Command(BaseCommand):
     @transaction.atomic
     def handle(self, *args, **options):
         # Create users
-        anon, admin, g, jebus = self.create_users()
+        anon, admin, g, user0 = self.create_users()
+
+        self.populate_common_external_databases()
 
         # Import Packages
         packages = [
@@ -169,7 +160,7 @@ class Command(BaseCommand):
         setting.save()
         setting.make_global_default()
 
-        for u in [anon, jebus]:
+        for u in [anon, user0]:
             u.default_setting = setting
             u.save()
 
@@ -200,6 +191,6 @@ class Command(BaseCommand):
         ml_model.build_model()
         # ml_model.evaluate_model()
 
-        # If available create EnviFormerModel
+        # If available, create EnviFormerModel
         if s.ENVIFORMER_PRESENT:
             enviFormer_model = EnviFormer.create(pack, 'EnviFormer - T0.5', 'EnviFormer Model with Threshold 0.5', 0.5)

epdb/management/commands/import_legacy_package.py

@@ -24,4 +24,4 @@ class Command(BaseCommand):
     def handle(self, *args, **options):
         owner = User.objects.get(username=options['owner'])
         package_data = json.load(open(options['data']))
-        PackageManager.import_package(package_data, owner)
+        PackageManager.import_legacy_package(package_data, owner)

epdb/models.py

@@ -578,32 +578,38 @@ class Package(EnviPathModel):
     license = models.ForeignKey('epdb.License', on_delete=models.SET_NULL, blank=True, null=True,
                                 verbose_name='License')
 
+    def delete(self, *args, **kwargs):
+        # explicitly handle related Rules
+        for r in self.rules.all():
+            r.delete()
+        super().delete(*args, **kwargs)
+
     def __str__(self):
         return f"{self.name} (pk={self.pk})"
 
     @property
     def compounds(self):
-        return Compound.objects.filter(package=self)
+        return self.compound_set.all()
 
     @property
     def rules(self):
-        return Rule.objects.filter(package=self)
+        return self.rule_set.all()
 
     @property
     def reactions(self):
-        return Reaction.objects.filter(package=self)
+        return self.reaction_set.all()
 
     @property
     def pathways(self) -> 'Pathway':
-        return Pathway.objects.filter(package=self)
+        return self.pathway_set.all()
 
     @property
     def scenarios(self):
-        return Scenario.objects.filter(package=self)
+        return self.scenario_set.all()
 
     @property
     def models(self):
-        return EPModel.objects.filter(package=self)
+        return self.epmodel_set.all()
 
     def _url(self):
         return '{}/package/{}'.format(s.SERVER_URL, self.uuid)
@@ -911,7 +917,6 @@ class CompoundStructure(EnviPathModel, AliasMixin, ScenarioMixin, ChemicalIdenti
 class Rule(PolymorphicModel, EnviPathModel, AliasMixin, ScenarioMixin):
     package = models.ForeignKey('epdb.Package', verbose_name='Package', on_delete=models.CASCADE, db_index=True)
 
-    # I think this only affects Django Admin which we are barely using
     # # https://github.com/django-polymorphic/django-polymorphic/issues/229
     # _non_polymorphic = models.Manager()
     #
@@ -1128,6 +1133,7 @@ class ParallelRule(Rule):
         return res
 
 
+
 class SequentialRule(Rule):
     simple_rules = models.ManyToManyField('epdb.SimpleRule', verbose_name='Simple rules',
                                           through='SequentialRuleOrdering')
@@ -1959,7 +1965,7 @@ class RuleBasedRelativeReasoning(PackageBasedModel):
         rbrr.package = package
 
         if name is None or name.strip() == '':
-            name = f"MLRelativeReasoning {RuleBasedRelativeReasoning.objects.filter(package=package).count() + 1}"
+            name = f"RuleBasedRelativeReasoning {RuleBasedRelativeReasoning.objects.filter(package=package).count() + 1}"
 
         rbrr.name = name
 

epdb/signals.py

@@ -1,8 +1,15 @@
+import logging
+import os
+import shutil
+
+from django.conf import settings as s
 from django.db import transaction
-from django.db.models.signals import pre_delete
+from django.db.models.signals import pre_delete, post_delete
 from django.dispatch import receiver
 
-from epdb.models import Node, Edge
+from epdb.models import Node, Edge, EPModel
+
+logger = logging.getLogger(__name__)
 
 
 @receiver(pre_delete, sender=Node)
@@ -18,3 +25,14 @@ def delete_orphan_edges(sender, instance, **kwargs):
         # check if the node that is about to be deleted is the only start node
         if edge.end_nodes.count() == 1:
             edge.delete()
+
+
+@receiver(post_delete, sender=EPModel)
+def delete_epmodel_files(sender, instance, **kwargs):
+    # Delete the files on disk for the deleted model
+    mod_uuid = str(instance.uuid)
+
+    for f in os.listdir(s.MODEL_DIR):
+        if f.startswith(mod_uuid):
+            logger.info(f"Deleting {os.path.join(s.MODEL_DIR, f)}")
+            shutil.rmtree(os.path.join(s.MODEL_DIR, f))

epdb/views.py

@@ -284,18 +284,21 @@ def packages(request):
 
         if hidden := request.POST.get('hidden', None):
 
-            if hidden == 'import-legacy-package-json':
+            if hidden in ['import-legacy-package-json', 'import-package-json']:
                 f =  request.FILES['file']
 
                 try:
                     file_data = f.read().decode("utf-8")
                     data = json.loads(file_data)
 
-                    pack = PackageManager.import_package(data, current_user)
+                    if hidden == 'import-legacy-package-json':
+                        pack = PackageManager.import_legacy_package(data, current_user)
+                    else:
+                        pack = PackageManager.import_pacakge(data, current_user)
+
                     return redirect(pack.url)
                 except UnicodeDecodeError:
                     return error(request, 'Invalid encoding.', f'Invalid encoding, must be UTF-8')
-
             else:
                 return HttpResponseBadRequest()
         else:
@@ -799,6 +802,15 @@ def package(request, package_uuid):
 
     if request.method == 'GET':
 
+        if request.GET.get("export", False) == "true":
+            filename = f"{current_package.name.replace(' ', '_')}_{current_package.uuid}.json"
+            pack_json = PackageManager.export_package(current_package, include_models=False,
+                                                      include_external_identifiers=False)
+            response = JsonResponse(pack_json, content_type='application/json')
+            response['Content-Disposition'] = f'attachment; filename="{filename}"'
+
+            return response
+
         context = get_base_context(request)
         context['title'] = f'enviPath - {current_package.name}'
 

pyproject.toml

@@ -12,6 +12,7 @@ dependencies = [
     "django-ninja>=1.4.1",
     "django-oauth-toolkit>=3.0.1",
     "django-polymorphic>=4.1.0",
+    "django-stubs>=5.2.4",
     "enviformer",
     "envipy-additional-information",
     "envipy-plugins",
@@ -33,4 +34,4 @@ envipy-plugins = { git = "ssh://git@git.envipath.com/enviPath/enviPy-plugins.git
 envipy-additional-information = { git = "ssh://git@git.envipath.com/enviPath/enviPy-additional-information.git", rev = "v0.1.4"}
 
 [project.optional-dependencies]
-ms-login = ["msal>=1.33.0"]
+ms-login = ["msal>=1.33.0"]

templates/actions/collections/package.html

@@ -2,7 +2,11 @@
     <a role="button" data-toggle="modal" data-target="#new_package_modal">
         <span class="glyphicon glyphicon-plus"></span> New Package</a>
 </li>
+<li>
+    <a role="button" data-toggle="modal" data-target="#import_package_modal">
+        <span class="glyphicon glyphicon-import"></span> Import Package from JSON</a>
+</li>
 <li>
     <a role="button" data-toggle="modal" data-target="#import_legacy_package_modal">
-        <span class="glyphicon glyphicon-import"></span> Import Package (Legacy)</a>
+        <span class="glyphicon glyphicon-import"></span> Import Package from legacy JSON</a>
 </li>

templates/actions/objects/package.html

@@ -11,6 +11,10 @@
         <a role="button" data-toggle="modal" data-target="#publish_package_modal">
             <i class="glyphicon glyphicon-bullhorn"></i> Publish Package</a>
     </li>
+    <li>
+        <a role="button" data-toggle="modal" data-target="#export_package_modal">
+            <i class="glyphicon glyphicon-bullhorn"></i> Export Package as JSON</a>
+    </li>
     <li>
         <a role="button" data-toggle="modal" data-target="#set_license_modal">
             <i class="glyphicon glyphicon-duplicate"></i> License</a>

templates/collections/objects_list.html

@@ -21,6 +21,7 @@
     {% block action_modals %}
         {% if object_type == 'package' %}
             {% include "modals/collections/new_package_modal.html" %}
+            {% include "modals/collections/import_package_modal.html" %}
             {% include "modals/collections/import_legacy_package_modal.html" %}
         {% elif object_type == 'compound' %}
             {% include "modals/collections/new_compound_modal.html" %}

templates/modals/collections/import_package_modal.html

@@ -0,0 +1,42 @@
+<div class="modal fade" tabindex="-1" id="import_package_modal" role="dialog"
+     aria-labelledby="import_package_modal" aria-hidden="true">
+    <div class="modal-dialog">
+        <div class="modal-content">
+            <div class="modal-header">
+                <button type="button" class="close" data-dismiss="modal">
+                    <span aria-hidden="true">&times;</span>
+                    <span class="sr-only">Close</span>
+                </button>
+                <h4 class="modal-title">Import Package</h4>
+            </div>
+            <div class="modal-body">
+                <p>Create a Package based on a JSON Export.</p>
+                <form id="import-package-modal-form" accept-charset="UTF-8" data-remote="true" method="post"
+                      enctype="multipart/form-data">
+                    {% csrf_token %}
+                    <p>
+                        <label class="btn btn-primary" for="jsonFile">
+                            <input id="jsonFile" name="file" type="file" style="display:none;"
+                                   onchange="$('#upload-file-info').html(this.files[0].name)">
+                            Choose JSON File
+                        </label>
+                        <span class="label label-info" id="upload-file-info"></span>
+                        <input type="hidden" value="import-package-json" name="hidden" readonly="">
+                    </p>
+                </form>
+            </div>
+            <div class="modal-footer">
+                <a id="import-package-modal-form-submit" class="btn btn-primary" href="#">Submit</a>
+                <button type="button" class="btn btn-default" data-dismiss="modal">Cancel</button>
+            </div>
+        </div>
+    </div>
+</div>
+<script>
+    $(function () {
+        $('#import-package-modal-form-submit').on('click', function (e) {
+            e.preventDefault();
+            $('#import-package-modal-form').submit();
+        });
+    });
+</script>

templates/modals/objects/export_package_modal.html

@@ -0,0 +1,36 @@
+{% load static %}
+<!-- Export Package -->
+<div id="export_package_modal" class="modal" tabindex="-1">
+    <div class="modal-dialog">
+        <div class="modal-content">
+            <div class="modal-header">
+                <h3 class="modal-title">Export Package as JSON</h3>
+                <button type="button" class="close" data-dismiss="modal" aria-label="Close">
+                    <span aria-hidden="true">&times;</span>
+                </button>
+            </div>
+            <div class="modal-body">
+                By clicking on Export the Package will be serialized into a JSON and directly downloaded.
+                <form id="export-package-modal-form" accept-charset="UTF-8" action="{{ package.url }}"
+                      data-remote="true" method="GET">
+                    <input type="hidden" name="export" value="true"/>
+                </form>
+            </div>
+            <div class="modal-footer">
+                <button type="button" class="btn btn-secondary" data-dismiss="modal">Close</button>
+                <button type="button" class="btn btn-primary" id="export-package-modal-form-submit">Export</button>
+            </div>
+        </div>
+    </div>
+</div>
+<script>
+    $(function () {
+
+        $('#export-package-modal-form-submit').click(function (e) {
+            e.preventDefault();
+            $('#export-package-modal-form').submit();
+            $('#export_package_modal').modal('hide');
+        });
+
+    })
+</script>

templates/objects/package.html

@@ -7,6 +7,7 @@
     {% include "modals/objects/edit_package_permissions_modal.html" %}
     {% include "modals/objects/publish_package_modal.html" %}
     {% include "modals/objects/set_license_modal.html" %}
+    {% include "modals/objects/export_package_modal.html" %}
     {% include "modals/objects/generic_delete_modal.html" %}
 {% endblock action_modals %}
 

tests/test_compound_model.py

@@ -5,7 +5,7 @@ from epdb.models import Compound, User, CompoundStructure
 
 
 class CompoundTest(TestCase):
-    fixtures = ["test_fixture.cleaned.json"]
+    fixtures = ["test_fixtures.json.gz"]
 
     def setUp(self):
         pass
@@ -16,13 +16,6 @@ class CompoundTest(TestCase):
         cls.user = User.objects.get(username='anonymous')
         cls.package = PackageManager.create_package(cls.user, 'Anon Test Package', 'No Desc')
 
-    @classmethod
-    def tearDownClass(cls):
-        pass
-
-    def tearDown(self):
-        pass
-
     def test_smoke(self):
         c = Compound.create(
             self.package,
@@ -78,7 +71,7 @@ class CompoundTest(TestCase):
             smiles='C1C(=NOC1(C2=CC(=CC(=C2)Cl)C(F)(F)F)C(F)(F)F)C3=CC=C(C4=CC=CC=C43)C(=O)NCC(=O)NCC(F)(F)F',
         )
 
-        self.assertEqual(c.name, 'no name')
+        self.assertEqual(c.name, 'Compound 1')
         self.assertEqual(c.description, 'no description')
 
     def test_empty_name_and_description_are_ignored(self):
@@ -89,7 +82,7 @@ class CompoundTest(TestCase):
             description='',
         )
 
-        self.assertEqual(c.name, 'no name')
+        self.assertEqual(c.name, 'Compound 1')
         self.assertEqual(c.description, 'no description')
 
     def test_deduplication(self):

tests/test_copy_objects.py

@@ -1,16 +1,13 @@
-import json
+from django.test import TestCase
 
 from django.test import TestCase
 
 from epdb.logic import PackageManager
-from epdb.models import Compound, User, CompoundStructure, Reaction, Rule, MLRelativeReasoning, Pathway
+from epdb.models import Compound, User, Reaction
 
 
 class CopyTest(TestCase):
-    fixtures = ["test_fixture.cleaned.json"]
-
-    def setUp(self):
-        pass
+    fixtures = ["test_fixtures.json.gz"]
 
     @classmethod
     def setUpClass(cls):
@@ -61,15 +58,6 @@ class CopyTest(TestCase):
             multi_step=False
         )
 
-
-    @classmethod
-    def tearDownClass(cls):
-        pass
-
-    def tearDown(self):
-        pass
-
-
     def test_compound_copy_basic(self):
         """Test basic compound copying functionality"""
         mapping = dict()
@@ -175,13 +163,12 @@ class CopyTest(TestCase):
         self.assertEqual(self.REACTION.multi_step, copied_reaction.multi_step)
         self.assertEqual(copied_reaction.package, self.target_package)
         self.assertEqual(self.REACTION.package, self.package)
-        
-        
+
     def test_reaction_copy_structures(self):
         """Test basic reaction copying functionality"""
         mapping = dict()
         copied_reaction = self.REACTION.copy(self.target_package, mapping)
-        
+
         for orig_educt, copy_educt in zip(self.REACTION.educts.all(), copied_reaction.educts.all()):
             self.assertNotEqual(orig_educt.uuid, copy_educt.uuid)
             self.assertEqual(orig_educt.name, copy_educt.name)
@@ -197,4 +184,3 @@ class CopyTest(TestCase):
             self.assertEqual(copy_product.compound.package, self.target_package)
             self.assertEqual(orig_product.compound.package, self.package)
             self.assertEqual(orig_product.smiles, copy_product.smiles)
-

tests/test_dataset.py

@@ -6,7 +6,7 @@ from utilities.ml import Dataset
 
 
 class DatasetTest(TestCase):
-    fixtures = ["test_fixture.cleaned.json"]
+    fixtures = ["test_fixtures.json.gz"]
 
     def setUp(self):
         self.cs1 = Compound.create(
@@ -38,7 +38,7 @@ class DatasetTest(TestCase):
 
     @classmethod
     def setUpClass(cls):
-        super(DatasetGeneratorTest, cls).setUpClass()
+        super(DatasetTest, cls).setUpClass()
         cls.user = User.objects.get(username='anonymous')
         cls.package = PackageManager.create_package(cls.user, 'Anon Test Package', 'No Desc')
 

tests/test_formatconverter.py

@@ -5,9 +5,6 @@ from utilities.chem import FormatConverter
 
 class FormatConverterTestCase(TestCase):
 
-    def setUp(self):
-        pass
-
     def test_standardization(self):
         smiles = 'C[n+]1c([N-](C))cccc1'
         standardized_smiles = FormatConverter.standardize(smiles)

tests/test_model.py

@@ -1,38 +1,27 @@
-import json
+from django.test import TestCase
 
 from django.test import TestCase
 
 from epdb.logic import PackageManager
-from epdb.models import Compound, User, CompoundStructure, Reaction, Rule, MLRelativeReasoning
+from epdb.models import User, MLRelativeReasoning, Package
 
 
 class ModelTest(TestCase):
-    fixtures = ["test_fixture.cleaned.json"]
-
-    def setUp(self):
-        pass
+    fixtures = ["test_fixtures.json.gz"]
 
     @classmethod
     def setUpClass(cls):
         super(ModelTest, cls).setUpClass()
         cls.user = User.objects.get(username='anonymous')
         cls.package = PackageManager.create_package(cls.user, 'Anon Test Package', 'No Desc')
-        bbd_data = json.load(open('fixtures/packages/2025-07-18/EAWAG-BBD.json'))
-        cls.BBD = PackageManager.import_package(bbd_data, cls.user)
-
-    @classmethod
-    def tearDownClass(cls):
-        pass
-
-    def tearDown(self):
-        pass
+        cls.BBD_SUBSET = Package.objects.get(name='Fixtures')
 
     def test_smoke(self):
         threshold = float(0.5)
 
         # get Package objects from urls
-        rule_package_objs = [self.BBD]
-        data_package_objs = [self.BBD]
+        rule_package_objs = [self.BBD_SUBSET]
+        data_package_objs = [self.BBD_SUBSET]
         eval_packages_objs = []
 
         mod = MLRelativeReasoning.create(
@@ -44,8 +33,8 @@ class ModelTest(TestCase):
             'ECC - BBD - 0.5',
             'Created MLRelativeReasoning in Testcase',
         )
-        ds = mod.load_dataset()
 
+        mod.build_dataset()
         mod.build_model()
         print("Model built!")
         mod.evaluate_model()

tests/test_reaction_model.py

@@ -1,14 +1,11 @@
 from django.test import TestCase
 
 from epdb.logic import PackageManager
-from epdb.models import Compound, User, CompoundStructure, Reaction, Rule
+from epdb.models import Compound, User, Reaction, Rule
 
 
 class ReactionTest(TestCase):
-    fixtures = ["test_fixture.cleaned.json"]
-
-    def setUp(self):
-        pass
+    fixtures = ["test_fixtures.json.gz"]
 
     @classmethod
     def setUpClass(cls):
@@ -16,13 +13,6 @@ class ReactionTest(TestCase):
         cls.user = User.objects.get(username='anonymous')
         cls.package = PackageManager.create_package(cls.user, 'Anon Test Package', 'No Desc')
 
-    @classmethod
-    def tearDownClass(cls):
-        pass
-
-    def tearDown(self):
-        pass
-
     def test_smoke(self):
         educt = Compound.create(
             self.package,
@@ -50,8 +40,6 @@ class ReactionTest(TestCase):
         self.assertEqual(r.name, 'Eawag BBD reaction r0001')
         self.assertEqual(r.description, 'no description')
 
-
-
     def test_string_educts_and_products(self):
         r = Reaction.create(
             package=self.package,

tests/test_rule_model.py

@@ -5,7 +5,7 @@ from epdb.models import Rule, User
 
 
 class RuleTest(TestCase):
-    fixtures = ["test_fixture.cleaned.json"]
+    fixtures = ["test_fixtures.json.gz"]
 
     def setUp(self):
         pass
@@ -16,13 +16,6 @@ class RuleTest(TestCase):
         cls.user = User.objects.get(username='anonymous')
         cls.package = PackageManager.create_package(cls.user, 'Anon Test Package', 'No Desc')
 
-    @classmethod
-    def tearDownClass(cls):
-        pass
-
-    def tearDown(self):
-        pass
-
     def test_smoke(self):
         r = Rule.create(
             rule_type='SimpleAmbitRule',

tests/test_ruleapplication.py

@@ -2,10 +2,10 @@ import gzip
 import json
 
 from django.conf import settings as s
-from django.test import TestCase
+from django.test import TestCase, tag
 from utilities.chem import FormatConverter
 
-
+@tag("slow")
 class RuleApplicationTest(TestCase):
 
     def setUp(self):
@@ -19,10 +19,12 @@ class RuleApplicationTest(TestCase):
 
     @classmethod
     def tearDownClass(cls):
+        super().tearDownClass()
         print(f"\nTotal Errors across Rules {len(cls.error_smiles)}")
         # print(cls.error_smiles)
 
     def tearDown(self):
+        super().tearDown()
         print(f"\nTotal errors {self.total_errors}")
 
     def run_bt_test(self, bt_rule_name):

tests/test_simpleambitrule.py

@@ -0,0 +1,362 @@
+from unittest.mock import patch, MagicMock, PropertyMock
+
+from django.test import TestCase
+
+from epdb.logic import PackageManager
+from epdb.models import User, SimpleAmbitRule
+
+
+class SimpleAmbitRuleTest(TestCase):
+    fixtures = ["test_fixtures.json.gz"]
+
+    @classmethod
+    def setUpClass(cls):
+        super(SimpleAmbitRuleTest, cls).setUpClass()
+        cls.user = User.objects.get(username='anonymous')
+        cls.package = PackageManager.create_package(cls.user, 'Simple Ambit Rule Test Package',
+                                                    'Test Package for SimpleAmbitRule')
+
+    def test_create_basic_rule(self):
+        """Test creating a basic SimpleAmbitRule with minimal parameters."""
+        smirks = '[H:5][C:1]([#6:6])([#1,#9,#17,#35,#53:4])[#9,#17,#35,#53]>>[H:5][C:1]([#6:6])([#8])[#1,#9,#17,#35,#53:4]'
+
+        rule = SimpleAmbitRule.create(
+            package=self.package,
+            smirks=smirks
+        )
+
+        self.assertIsInstance(rule, SimpleAmbitRule)
+        self.assertEqual(rule.smirks, smirks)
+        self.assertEqual(rule.package, self.package)
+        self.assertRegex(rule.name, r'Rule \d+')
+        self.assertEqual(rule.description, 'no description')
+        self.assertIsNone(rule.reactant_filter_smarts)
+        self.assertIsNone(rule.product_filter_smarts)
+
+    def test_create_with_all_parameters(self):
+        """Test creating SimpleAmbitRule with all parameters."""
+        smirks = '[H:5][C:1]([#6:6])([#1,#9,#17,#35,#53:4])[#9,#17,#35,#53]>>[H:5][C:1]([#6:6])([#8])[#1,#9,#17,#35,#53:4]'
+        name = 'Test Rule'
+        description = 'A test biotransformation rule'
+        reactant_filter = '[CH2X4]'
+        product_filter = '[OH]'
+
+        rule = SimpleAmbitRule.create(
+            package=self.package,
+            name=name,
+            description=description,
+            smirks=smirks,
+            reactant_filter_smarts=reactant_filter,
+            product_filter_smarts=product_filter
+        )
+
+        self.assertEqual(rule.name, name)
+        self.assertEqual(rule.description, description)
+        self.assertEqual(rule.smirks, smirks)
+        self.assertEqual(rule.reactant_filter_smarts, reactant_filter)
+        self.assertEqual(rule.product_filter_smarts, product_filter)
+
+    def test_smirks_required(self):
+        """Test that SMIRKS is required for rule creation."""
+        with self.assertRaises(ValueError) as cm:
+            SimpleAmbitRule.create(package=self.package, smirks=None)
+        self.assertIn('SMIRKS is required', str(cm.exception))
+
+        with self.assertRaises(ValueError) as cm:
+            SimpleAmbitRule.create(package=self.package, smirks='')
+        self.assertIn('SMIRKS is required', str(cm.exception))
+
+        with self.assertRaises(ValueError) as cm:
+            SimpleAmbitRule.create(package=self.package, smirks='   ')
+        self.assertIn('SMIRKS is required', str(cm.exception))
+
+    @patch('epdb.models.FormatConverter.is_valid_smirks')
+    def test_invalid_smirks_validation(self, mock_is_valid):
+        """Test validation of SMIRKS format."""
+        mock_is_valid.return_value = False
+
+        invalid_smirks = 'invalid_smirks_string'
+        with self.assertRaises(ValueError) as cm:
+            SimpleAmbitRule.create(
+                package=self.package,
+                smirks=invalid_smirks
+            )
+
+        self.assertIn(f'SMIRKS "{invalid_smirks}" is invalid', str(cm.exception))
+        mock_is_valid.assert_called_once_with(invalid_smirks)
+
+    def test_smirks_trimming(self):
+        """Test that SMIRKS strings are trimmed during creation."""
+        smirks = '[H:5][C:1]([#6:6])([#1,#9,#17,#35,#53:4])[#9,#17,#35,#53]>>[H:5][C:1]([#6:6])([#8])[#1,#9,#17,#35,#53:4]'
+        smirks_with_whitespace = f'   {smirks}   '
+
+        rule = SimpleAmbitRule.create(
+            package=self.package,
+            smirks=smirks_with_whitespace
+        )
+
+        self.assertEqual(rule.smirks, smirks)
+
+    def test_empty_name_and_description_handling(self):
+        """Test that empty name and description are handled appropriately."""
+        smirks = '[H:5][C:1]([#6:6])([#1,#9,#17,#35,#53:4])[#9,#17,#35,#53]>>[H:5][C:1]([#6:6])([#8])[#1,#9,#17,#35,#53:4]'
+
+        rule = SimpleAmbitRule.create(
+            package=self.package,
+            smirks=smirks,
+            name='',
+            description='   '
+        )
+
+        self.assertRegex(rule.name, r'Rule \d+')
+        self.assertEqual(rule.description, 'no description')
+
+    def test_deduplication_basic(self):
+        """Test that identical rules are deduplicated."""
+        smirks = '[H:5][C:1]([#6:6])([#1,#9,#17,#35,#53:4])[#9,#17,#35,#53]>>[H:5][C:1]([#6:6])([#8])[#1,#9,#17,#35,#53:4]'
+
+        rule1 = SimpleAmbitRule.create(
+            package=self.package,
+            smirks=smirks,
+            name='Rule 1'
+        )
+
+        rule2 = SimpleAmbitRule.create(
+            package=self.package,
+            smirks=smirks,
+            name='Rule 2'  # Different name, but same SMIRKS
+        )
+
+        self.assertEqual(rule1.pk, rule2.pk)
+        self.assertEqual(SimpleAmbitRule.objects.filter(package=self.package, smirks=smirks).count(), 1)
+
+    def test_deduplication_with_filters(self):
+        """Test deduplication with filter SMARTS."""
+        smirks = '[H:5][C:1]([#6:6])([#1,#9,#17,#35,#53:4])[#9,#17,#35,#53]>>[H:5][C:1]([#6:6])([#8])[#1,#9,#17,#35,#53:4]'
+        reactant_filter = '[CH2X4]'
+        product_filter = '[OH]'
+
+        rule1 = SimpleAmbitRule.create(
+            package=self.package,
+            smirks=smirks,
+            reactant_filter_smarts=reactant_filter,
+            product_filter_smarts=product_filter
+        )
+
+        rule2 = SimpleAmbitRule.create(
+            package=self.package,
+            smirks=smirks,
+            reactant_filter_smarts=reactant_filter,
+            product_filter_smarts=product_filter
+        )
+
+        self.assertEqual(rule1.pk, rule2.pk)
+
+    def test_no_deduplication_different_filters(self):
+        """Test that rules with different filters are not deduplicated."""
+        smirks = '[H:5][C:1]([#6:6])([#1,#9,#17,#35,#53:4])[#9,#17,#35,#53]>>[H:5][C:1]([#6:6])([#8])[#1,#9,#17,#35,#53:4]'
+
+        rule1 = SimpleAmbitRule.create(
+            package=self.package,
+            smirks=smirks,
+            reactant_filter_smarts='[CH2X4]'
+        )
+
+        rule2 = SimpleAmbitRule.create(
+            package=self.package,
+            smirks=smirks,
+            reactant_filter_smarts='[CH3X4]'
+        )
+
+        self.assertNotEqual(rule1.pk, rule2.pk)
+        self.assertEqual(SimpleAmbitRule.objects.filter(package=self.package, smirks=smirks).count(), 2)
+
+    def test_filter_smarts_trimming(self):
+        """Test that filter SMARTS are trimmed and handled correctly."""
+        smirks = '[H:5][C:1]([#6:6])([#1,#9,#17,#35,#53:4])[#9,#17,#35,#53]>>[H:5][C:1]([#6:6])([#8])[#1,#9,#17,#35,#53:4]'
+
+        # Test with whitespace-only filters (should be treated as None)
+        rule = SimpleAmbitRule.create(
+            package=self.package,
+            smirks=smirks,
+            reactant_filter_smarts='   ',
+            product_filter_smarts='   '
+        )
+
+        self.assertIsNone(rule.reactant_filter_smarts)
+        self.assertIsNone(rule.product_filter_smarts)
+
+    def test_url_property(self):
+        """Test the URL property generation."""
+        rule = SimpleAmbitRule.create(
+            package=self.package,
+            smirks='[H:1][C:2]>>[H:1][O:2]'
+        )
+
+        expected_url = f'{self.package.url}/simple-ambit-rule/{rule.uuid}'
+        self.assertEqual(rule.url, expected_url)
+
+    @patch('epdb.models.FormatConverter.apply')
+    def test_apply_method(self, mock_apply):
+        """Test the apply method delegates to FormatConverter."""
+        mock_apply.return_value = ['product1', 'product2']
+
+        rule = SimpleAmbitRule.create(
+            package=self.package,
+            smirks='[H:1][C:2]>>[H:1][O:2]'
+        )
+
+        test_smiles = 'CCO'
+        result = rule.apply(test_smiles)
+
+        mock_apply.assert_called_once_with(test_smiles, rule.smirks)
+        self.assertEqual(result, ['product1', 'product2'])
+
+    def test_reactants_smarts_property(self):
+        """Test reactants_smarts property extracts correct part of SMIRKS."""
+        smirks = '[H:5][C:1]([#6:6])([#1,#9,#17,#35,#53:4])[#9,#17,#35,#53]>>[H:5][C:1]([#6:6])([#8])[#1,#9,#17,#35,#53:4]'
+        expected_reactants = '[H:5][C:1]([#6:6])([#1,#9,#17,#35,#53:4])[#9,#17,#35,#53]'
+
+        rule = SimpleAmbitRule.create(
+            package=self.package,
+            smirks=smirks
+        )
+
+        self.assertEqual(rule.reactants_smarts, expected_reactants)
+
+    def test_products_smarts_property(self):
+        """Test products_smarts property extracts correct part of SMIRKS."""
+        smirks = '[H:5][C:1]([#6:6])([#1,#9,#17,#35,#53:4])[#9,#17,#35,#53]>>[H:5][C:1]([#6:6])([#8])[#1,#9,#17,#35,#53:4]'
+        expected_products = '[H:5][C:1]([#6:6])([#8])[#1,#9,#17,#35,#53:4]'
+
+        rule = SimpleAmbitRule.create(
+            package=self.package,
+            smirks=smirks
+        )
+
+        self.assertEqual(rule.products_smarts, expected_products)
+
+    @patch('epdb.models.Package.objects')
+    def test_related_reactions_property(self, mock_package_objects):
+        """Test related_reactions property returns correct queryset."""
+        mock_qs = MagicMock()
+        mock_package_objects.filter.return_value = mock_qs
+
+        rule = SimpleAmbitRule.create(
+            package=self.package,
+            smirks='[H:1][C:2]>>[H:1][O:2]'
+        )
+
+        # Instead of directly assigning, patch the property or use with patch.object
+        with patch.object(type(rule), 'reaction_rule', new_callable=PropertyMock) as mock_reaction_rule:
+            mock_reaction_rule.return_value.filter.return_value.order_by.return_value = ['reaction1', 'reaction2']
+
+            result = rule.related_reactions
+
+            mock_package_objects.filter.assert_called_once_with(reviewed=True)
+            mock_reaction_rule.return_value.filter.assert_called_once_with(package__in=mock_qs)
+            mock_reaction_rule.return_value.filter.return_value.order_by.assert_called_once_with('name')
+            self.assertEqual(result, ['reaction1', 'reaction2'])
+
+    @patch('epdb.models.Pathway.objects')
+    @patch('epdb.models.Edge.objects')
+    def test_related_pathways_property(self, mock_edge_objects, mock_pathway_objects):
+        """Test related_pathways property returns correct queryset."""
+
+        mock_related_reactions = ['reaction1', 'reaction2']
+
+        with patch.object(SimpleAmbitRule, "related_reactions", new_callable=PropertyMock) as mock_prop:
+            mock_prop.return_value = mock_related_reactions
+
+            rule = SimpleAmbitRule.create(
+                package=self.package,
+                smirks='[H:1][C:2]>>[H:1][O:2]'
+            )
+
+            # Mock Edge objects query
+            mock_edge_values = MagicMock()
+            mock_edge_values.values.return_value = ['pathway_id1', 'pathway_id2']
+            mock_edge_objects.filter.return_value = mock_edge_values
+
+            # Mock Pathway objects query
+            mock_pathway_qs = MagicMock()
+            mock_pathway_objects.filter.return_value.order_by.return_value = mock_pathway_qs
+
+            result = rule.related_pathways
+
+            mock_edge_objects.filter.assert_called_once_with(edge_label__in=mock_related_reactions)
+            mock_edge_values.values.assert_called_once_with('pathway_id')
+            mock_pathway_objects.filter.assert_called_once()
+            self.assertEqual(result, mock_pathway_qs)
+
+    @patch('epdb.models.IndigoUtils.smirks_to_svg')
+    def test_as_svg_property(self, mock_smirks_to_svg):
+        """Test as_svg property calls IndigoUtils correctly."""
+        mock_smirks_to_svg.return_value = '<svg>test_svg</svg>'
+
+        rule = SimpleAmbitRule.create(
+            package=self.package,
+            smirks='[H:1][C:2]>>[H:1][O:2]'
+        )
+
+        result = rule.as_svg
+        mock_smirks_to_svg.assert_called_once_with(rule.smirks, True, width=800, height=400)
+        self.assertEqual(result, '<svg>test_svg</svg>')
+
+    def test_atomic_transaction(self):
+        """Test that rule creation is atomic."""
+        smirks = '[H:1][C:2]>>[H:1][O:2]'
+
+        # This should work normally
+        rule = SimpleAmbitRule.create(package=self.package, smirks=smirks)
+        self.assertIsInstance(rule, SimpleAmbitRule)
+
+        # Test transaction rollback on error
+        with patch('epdb.models.SimpleAmbitRule.save', side_effect=Exception('Database error')):
+            with self.assertRaises(Exception):
+                SimpleAmbitRule.create(package=self.package, smirks='[H:3][C:4]>>[H:3][O:4]')
+
+        # Verify no partial data was saved
+        self.assertEqual(SimpleAmbitRule.objects.filter(package=self.package).count(), 1)
+
+    def test_multiple_duplicate_warning(self):
+        """Test logging when multiple duplicates are found."""
+        smirks = '[H:1][C:2]>>[H:1][O:2]'
+
+        # Create first rule
+        rule1 = SimpleAmbitRule.create(package=self.package, smirks=smirks)
+
+        # Manually create a duplicate to simulate the error condition
+        rule2 = SimpleAmbitRule(package=self.package, smirks=smirks, name='Manual Rule')
+        rule2.save()
+
+        with patch('epdb.models.logger') as mock_logger:
+            # This should find the existing rule and log an error about multiple matches
+            result = SimpleAmbitRule.create(package=self.package, smirks=smirks)
+
+            # Should return the first matching rule
+            self.assertEqual(result.pk, rule1.pk)
+
+            # Should log an error about multiple matches
+            mock_logger.error.assert_called()
+            self.assertIn('More than one rule matched', mock_logger.error.call_args[0][0])
+
+    def test_model_fields(self):
+        """Test model field properties."""
+        rule = SimpleAmbitRule.create(
+            package=self.package,
+            smirks='[H:1][C:2]>>[H:1][O:2]',
+            reactant_filter_smarts='[CH3]',
+            product_filter_smarts='[OH]'
+        )
+
+        # Test field properties
+        self.assertFalse(rule._meta.get_field('smirks').blank)
+        self.assertFalse(rule._meta.get_field('smirks').null)
+        self.assertTrue(rule._meta.get_field('reactant_filter_smarts').null)
+        self.assertTrue(rule._meta.get_field('product_filter_smarts').null)
+
+        # Test verbose names
+        self.assertEqual(rule._meta.get_field('smirks').verbose_name, 'SMIRKS')
+        self.assertEqual(rule._meta.get_field('reactant_filter_smarts').verbose_name, 'Reactant Filter SMARTS')
+        self.assertEqual(rule._meta.get_field('product_filter_smarts').verbose_name, 'Product Filter SMARTS')

utilities/misc.py

@@ -1,14 +1,29 @@
+import base64
+import hashlib
+import hmac
 import html
+import json
 import logging
+import uuid
 from collections import defaultdict
+from datetime import datetime
 from enum import Enum
 from types import NoneType
-from typing import Dict, List, Any
+from typing import Dict, Any, List
 
+from django.db import transaction
 from envipy_additional_information import Interval, EnviPyModel
 from envipy_additional_information import NAME_MAPPING
 from pydantic import BaseModel, HttpUrl
 
+from epdb.models import (
+    Package, Compound, CompoundStructure, SimpleRule, SimpleAmbitRule,
+    SimpleRDKitRule, ParallelRule, SequentialRule, Reaction, Pathway, Node, Edge, Scenario, EPModel,
+    MLRelativeReasoning,
+    RuleBasedRelativeReasoning, EnviFormer, PluginModel, ExternalIdentifier,
+    ExternalDatabase, License
+)
+
 logger = logging.getLogger(__name__)
 
 
@@ -166,3 +181,898 @@ class HTMLGenerator:
                 instances[class_name].append(instance)
 
         return instances
+
+
+class PackageExporter:
+
+    def __init__(self, package: Package, include_models: bool = False, include_external_identifiers: bool = True):
+        self._raw_package = package
+        self.include_modes = include_models
+        self.include_external_identifiers = include_external_identifiers
+
+    def do_export(self):
+        return PackageExporter._export_package_as_json(self._raw_package, self.include_modes, self.include_external_identifiers)
+
+    @staticmethod
+    def _export_package_as_json(package: Package, include_models: bool = False,
+                               include_external_identifiers: bool = True) -> Dict[str, Any]:
+        """
+        Dumps a Package and all its related objects as JSON.
+
+        Args:
+            package: The Package instance to dump
+            include_models: Whether to include EPModel objects
+            include_external_identifiers: Whether to include external identifiers
+
+        Returns:
+            Dict containing the complete package data as JSON-serializable structure
+        """
+
+        def serialize_base_object(obj, include_aliases: bool = True, include_scenarios: bool = True) -> Dict[str, Any]:
+            """Serialize common EnviPathModel fields"""
+            base_dict = {
+                'uuid': str(obj.uuid),
+                'name': obj.name,
+                'description': obj.description,
+                'url': obj.url,
+                'kv': obj.kv,
+            }
+
+            # Add aliases if the object has them
+            if include_aliases and hasattr(obj, 'aliases'):
+                base_dict['aliases'] = obj.aliases
+
+            # Add scenarios if the object has them
+            if include_scenarios and hasattr(obj, 'scenarios'):
+                base_dict['scenarios'] = [
+                    {'uuid': str(s.uuid), 'url': s.url} for s in obj.scenarios.all()
+                ]
+
+            return base_dict
+
+        def serialize_external_identifiers(obj) -> List[Dict[str, Any]]:
+            """Serialize external identifiers for an object"""
+            if not include_external_identifiers or not hasattr(obj, 'external_identifiers'):
+                return []
+
+            identifiers = []
+            for ext_id in obj.external_identifiers.all():
+                identifier_dict = {
+                    'uuid': str(ext_id.uuid),
+                    'database': {
+                        'uuid': str(ext_id.database.uuid),
+                        'name': ext_id.database.name,
+                        'base_url': ext_id.database.base_url
+                    },
+                    'identifier_value': ext_id.identifier_value,
+                    'url': ext_id.url,
+                    'is_primary': ext_id.is_primary
+                }
+                identifiers.append(identifier_dict)
+            return identifiers
+
+        # Start with the package itself
+        result = serialize_base_object(package, include_aliases=True, include_scenarios=True)
+        result['reviewed'] = package.reviewed
+
+        # # Add license information
+        # if package.license:
+        #     result['license'] = {
+        #         'uuid': str(package.license.uuid),
+        #         'name': package.license.name,
+        #         'link': package.license.link,
+        #         'image_link': package.license.image_link
+        #     }
+        # else:
+        #     result['license'] = None
+
+        # Initialize collections
+        result.update({
+            'compounds': [],
+            'structures': [],
+            'rules': {
+                'simple_rules': [],
+                'parallel_rules': [],
+                'sequential_rules': []
+            },
+            'reactions': [],
+            'pathways': [],
+            'nodes': [],
+            'edges': [],
+            'scenarios': [],
+            'models': []
+        })
+
+        print(f"Exporting package: {package.name}")
+
+        # Export compounds
+        print("Exporting compounds...")
+        for compound in package.compounds.prefetch_related('default_structure').order_by('url'):
+            compound_dict = serialize_base_object(compound, include_aliases=True, include_scenarios=True)
+
+            if compound.default_structure:
+                compound_dict['default_structure'] = {
+                    'uuid': str(compound.default_structure.uuid),
+                    'url': compound.default_structure.url
+                }
+            else:
+                compound_dict['default_structure'] = None
+
+            compound_dict['external_identifiers'] = serialize_external_identifiers(compound)
+            result['compounds'].append(compound_dict)
+
+        # Export compound structures
+        print("Exporting compound structures...")
+        compound_structures = CompoundStructure.objects.filter(
+            compound__package=package
+        ).select_related('compound').order_by('url')
+
+        for structure in compound_structures:
+            structure_dict = serialize_base_object(structure, include_aliases=True, include_scenarios=True)
+            structure_dict.update({
+                'compound': {
+                    'uuid': str(structure.compound.uuid),
+                    'url': structure.compound.url
+                },
+                'smiles': structure.smiles,
+                'canonical_smiles': structure.canonical_smiles,
+                'inchikey': structure.inchikey,
+                'normalized_structure': structure.normalized_structure,
+                'external_identifiers': serialize_external_identifiers(structure)
+            })
+            result['structures'].append(structure_dict)
+
+        # Export rules
+        print("Exporting rules...")
+
+        # Simple rules (including SimpleAmbitRule and SimpleRDKitRule)
+        for rule in SimpleRule.objects.filter(package=package).order_by('url'):
+            rule_dict = serialize_base_object(rule, include_aliases=True, include_scenarios=True)
+
+            # Add specific fields for SimpleAmbitRule
+            if isinstance(rule, SimpleAmbitRule):
+                rule_dict.update({
+                    'rule_type': 'SimpleAmbitRule',
+                    'smirks': rule.smirks,
+                    'reactant_filter_smarts': rule.reactant_filter_smarts or '',
+                    'product_filter_smarts': rule.product_filter_smarts or ''
+                })
+            elif isinstance(rule, SimpleRDKitRule):
+                rule_dict.update({
+                    'rule_type': 'SimpleRDKitRule',
+                    'reaction_smarts': rule.reaction_smarts
+                })
+            else:
+                rule_dict['rule_type'] = 'SimpleRule'
+
+            result['rules']['simple_rules'].append(rule_dict)
+
+        # Parallel rules
+        for rule in ParallelRule.objects.filter(package=package).prefetch_related('simple_rules').order_by('url'):
+            rule_dict = serialize_base_object(rule, include_aliases=True, include_scenarios=True)
+            rule_dict['rule_type'] = 'ParallelRule'
+            rule_dict['simple_rules'] = [
+                {'uuid': str(sr.uuid), 'url': sr.url} for sr in rule.simple_rules.all()
+            ]
+            result['rules']['parallel_rules'].append(rule_dict)
+
+        # Sequential rules
+        for rule in SequentialRule.objects.filter(package=package).prefetch_related('simple_rules').order_by('url'):
+            rule_dict = serialize_base_object(rule, include_aliases=True, include_scenarios=True)
+            rule_dict['rule_type'] = 'SequentialRule'
+            rule_dict['simple_rules'] = [
+                {'uuid': str(sr.uuid), 'url': sr.url,
+                 'order_index': sr.sequentialruleordering_set.get(sequential_rule=rule).order_index}
+                for sr in rule.simple_rules.all()
+            ]
+            result['rules']['sequential_rules'].append(rule_dict)
+
+        # Export reactions
+        print("Exporting reactions...")
+        for reaction in package.reactions.prefetch_related('educts', 'products', 'rules').order_by('url'):
+            reaction_dict = serialize_base_object(reaction, include_aliases=True, include_scenarios=True)
+            reaction_dict.update({
+                'educts': [{'uuid': str(e.uuid), 'url': e.url} for e in reaction.educts.all()],
+                'products': [{'uuid': str(p.uuid), 'url': p.url} for p in reaction.products.all()],
+                'rules': [{'uuid': str(r.uuid), 'url': r.url} for r in reaction.rules.all()],
+                'multi_step': reaction.multi_step,
+                'medline_references': reaction.medline_references,
+                'external_identifiers': serialize_external_identifiers(reaction)
+            })
+            result['reactions'].append(reaction_dict)
+
+        # Export pathways
+        print("Exporting pathways...")
+        for pathway in package.pathways.order_by('url'):
+            pathway_dict = serialize_base_object(pathway, include_aliases=True, include_scenarios=True)
+
+            # Add setting reference if exists
+            if hasattr(pathway, 'setting') and pathway.setting:
+                pathway_dict['setting'] = {
+                    'uuid': str(pathway.setting.uuid),
+                    'url': pathway.setting.url
+                }
+            else:
+                pathway_dict['setting'] = None
+
+            result['pathways'].append(pathway_dict)
+
+        # Export nodes
+        print("Exporting nodes...")
+        pathway_nodes = Node.objects.filter(
+            pathway__package=package
+        ).select_related('pathway', 'default_node_label').prefetch_related('node_labels', 'out_edges').order_by('url')
+
+        for node in pathway_nodes:
+            node_dict = serialize_base_object(node, include_aliases=True, include_scenarios=True)
+            node_dict.update({
+                'pathway': {'uuid': str(node.pathway.uuid), 'url': node.pathway.url},
+                'default_node_label': {
+                    'uuid': str(node.default_node_label.uuid),
+                    'url': node.default_node_label.url
+                },
+                'node_labels': [
+                    {'uuid': str(label.uuid), 'url': label.url} for label in node.node_labels.all()
+                ],
+                'out_edges': [
+                    {'uuid': str(edge.uuid), 'url': edge.url} for edge in node.out_edges.all()
+                ],
+                'depth': node.depth
+            })
+            result['nodes'].append(node_dict)
+
+        # Export edges
+        print("Exporting edges...")
+        pathway_edges = Edge.objects.filter(
+            pathway__package=package
+        ).select_related('pathway', 'edge_label').prefetch_related('start_nodes', 'end_nodes').order_by('url')
+
+        for edge in pathway_edges:
+            edge_dict = serialize_base_object(edge, include_aliases=True, include_scenarios=True)
+            edge_dict.update({
+                'pathway': {'uuid': str(edge.pathway.uuid), 'url': edge.pathway.url},
+                'edge_label': {'uuid': str(edge.edge_label.uuid), 'url': edge.edge_label.url},
+                'start_nodes': [
+                    {'uuid': str(node.uuid), 'url': node.url} for node in edge.start_nodes.all()
+                ],
+                'end_nodes': [
+                    {'uuid': str(node.uuid), 'url': node.url} for node in edge.end_nodes.all()
+                ]
+            })
+            result['edges'].append(edge_dict)
+
+        # Export scenarios
+        print("Exporting scenarios...")
+        for scenario in package.scenarios.order_by('url'):
+            scenario_dict = serialize_base_object(scenario, include_aliases=False, include_scenarios=False)
+            scenario_dict.update({
+                'scenario_date': scenario.scenario_date,
+                'scenario_type': scenario.scenario_type,
+                'parent': {
+                    'uuid': str(scenario.parent.uuid),
+                    'url': scenario.parent.url
+                } if scenario.parent else None,
+                'additional_information': scenario.additional_information
+            })
+            result['scenarios'].append(scenario_dict)
+
+        # Export models
+        if include_models:
+            print("Exporting models...")
+            package_models = package.models.select_related('app_domain').prefetch_related(
+                'rule_packages', 'data_packages', 'eval_packages'
+            ).order_by('url')
+
+            for model in package_models:
+                model_dict = serialize_base_object(model, include_aliases=True, include_scenarios=False)
+
+                # Common fields for PackageBasedModel
+                if hasattr(model, 'rule_packages'):
+                    model_dict.update({
+                        'rule_packages': [
+                            {'uuid': str(p.uuid), 'url': p.url} for p in model.rule_packages.all()
+                        ],
+                        'data_packages': [
+                            {'uuid': str(p.uuid), 'url': p.url} for p in model.data_packages.all()
+                        ],
+                        'eval_packages': [
+                            {'uuid': str(p.uuid), 'url': p.url} for p in model.eval_packages.all()
+                        ],
+                        'threshold': model.threshold,
+                        'eval_results': model.eval_results,
+                        'model_status': model.model_status
+                    })
+
+                    if model.app_domain:
+                        model_dict['app_domain'] = {
+                            'uuid': str(model.app_domain.uuid),
+                            'url': model.app_domain.url
+                        }
+                    else:
+                        model_dict['app_domain'] = None
+
+                # Specific fields for different model types
+                if isinstance(model, RuleBasedRelativeReasoning):
+                    model_dict.update({
+                        'model_type': 'RuleBasedRelativeReasoning',
+                        'min_count': model.min_count,
+                        'max_count': model.max_count
+                    })
+                elif isinstance(model, MLRelativeReasoning):
+                    model_dict['model_type'] = 'MLRelativeReasoning'
+                elif isinstance(model, EnviFormer):
+                    model_dict['model_type'] = 'EnviFormer'
+                elif isinstance(model, PluginModel):
+                    model_dict['model_type'] = 'PluginModel'
+                else:
+                    model_dict['model_type'] = 'EPModel'
+
+                result['models'].append(model_dict)
+
+        print(f"Export completed for package: {package.name}")
+        print(f"- Compounds: {len(result['compounds'])}")
+        print(f"- Structures: {len(result['structures'])}")
+        print(f"- Simple rules: {len(result['rules']['simple_rules'])}")
+        print(f"- Parallel rules: {len(result['rules']['parallel_rules'])}")
+        print(f"- Sequential rules: {len(result['rules']['sequential_rules'])}")
+        print(f"- Reactions: {len(result['reactions'])}")
+        print(f"- Pathways: {len(result['pathways'])}")
+        print(f"- Nodes: {len(result['nodes'])}")
+        print(f"- Edges: {len(result['edges'])}")
+        print(f"- Scenarios: {len(result['scenarios'])}")
+        print(f"- Models: {len(result['models'])}")
+
+        return result
+
+class PackageImporter:
+    """
+    Imports package data from JSON export.
+    Handles object creation, relationship mapping, and dependency resolution.
+    """
+
+    def __init__(self, package: Dict[str, Any], preserve_uuids: bool = False, add_import_timestamp=True,
+                 trust_reviewed=False):
+        """
+        Initialize the importer.
+
+        Args:
+            preserve_uuids: If True, preserve original UUIDs. If False, generate new ones.
+        """
+        self.preserve_uuids = preserve_uuids
+        self.add_import_timestamp = add_import_timestamp
+        self.trust_reviewed = trust_reviewed
+        self.uuid_mapping = {}
+        self.object_cache = {}
+        self._raw_package = package
+
+    def _get_or_generate_uuid(self, original_uuid: str) -> str:
+        """Get mapped UUID or generate new one if not preserving UUIDs."""
+        if self.preserve_uuids:
+            return original_uuid
+
+        if original_uuid not in self.uuid_mapping:
+            self.uuid_mapping[original_uuid] = str(uuid.uuid4())
+
+        return self.uuid_mapping[original_uuid]
+
+    def _cache_object(self, model_name: str, uuid_str: str, obj):
+        """Cache a created object for later reference."""
+        self.object_cache[(model_name, uuid_str)] = obj
+
+    def _get_cached_object(self, model_name: str, uuid_str: str):
+        """Get a cached object by model name and UUID."""
+        return self.object_cache.get((model_name, uuid_str))
+
+    def do_import(self) -> Package:
+        return self._import_package_from_json(self._raw_package)
+
+    @staticmethod
+    def sign(data: Dict[str, Any], key: str) -> Dict[str, Any]:
+        json_str = json.dumps(data, sort_keys=True, separators=(",", ":"))
+        signature = hmac.new(key.encode(), json_str.encode(), hashlib.sha256).digest()
+        data['_signature'] = base64.b64encode(signature).decode()
+        return data
+
+    @staticmethod
+    def verify(data: Dict[str, Any], key: str) -> bool:
+        copied_data = data.copy()
+        sig = copied_data.pop("_signature")
+        signature = base64.b64decode(sig, validate=True)
+        json_str = json.dumps(copied_data, sort_keys=True, separators=(",", ":"))
+        expected = hmac.new(key.encode(), json_str.encode(), hashlib.sha256).digest()
+        return hmac.compare_digest(signature, expected)
+
+
+    @transaction.atomic
+    def _import_package_from_json(self, package_data: Dict[str, Any]) -> Package:
+        """
+        Import a complete package from JSON data.
+
+        Args:
+            package_data: Dictionary containing the package export data
+
+        Returns:
+            The created Package instance
+        """
+        print(f"Starting import of package: {package_data['name']}")
+
+        # Create the main package
+        package = self._create_package(package_data)
+
+        # Import in dependency order
+        self._import_compounds(package, package_data.get('compounds', []))
+        self._import_structures(package, package_data.get('structures', []))
+        self._import_rules(package, package_data.get('rules', {}))
+        self._import_reactions(package, package_data.get('reactions', []))
+        self._import_pathways(package, package_data.get('pathways', []))
+        self._import_nodes(package, package_data.get('nodes', []))
+        self._import_edges(package, package_data.get('edges', []))
+        self._import_scenarios(package, package_data.get('scenarios', []))
+
+        if package_data.get('models'):
+            self._import_models(package, package_data['models'])
+
+        # Set default structures for compounds (after all structures are created)
+        self._set_default_structures(package_data.get('compounds', []))
+
+        print(f"Package import completed: {package.name}")
+        return package
+
+    def _create_package(self, package_data: Dict[str, Any]) -> Package:
+        """Create the main package object."""
+        package_uuid = self._get_or_generate_uuid(package_data['uuid'])
+
+        # Handle license
+        license_obj = None
+        if package_data.get('license'):
+            license_data = package_data['license']
+            license_obj, _ = License.objects.get_or_create(
+                name=license_data['name'],
+                defaults={
+                    'link': license_data.get('link', ''),
+                    'image_link': license_data.get('image_link', '')
+                }
+            )
+
+        new_name = package_data.get('name')
+        if self.add_import_timestamp:
+            new_name = f"{new_name} - Imported at {datetime.now()}"
+
+        new_reviewed = False
+        if self.trust_reviewed:
+            new_reviewed = package_data.get('reviewed', False)
+
+        package = Package.objects.create(
+            uuid=package_uuid,
+            name=new_name,
+            description=package_data['description'],
+            kv=package_data.get('kv', {}),
+            reviewed=new_reviewed,
+            license=license_obj
+        )
+
+        self._cache_object('Package', package_data['uuid'], package)
+        print(f"Created package: {package.name}")
+        return package
+
+    def _import_compounds(self, package: Package, compounds_data: List[Dict[str, Any]]):
+        """Import compounds."""
+        print(f"Importing {len(compounds_data)} compounds...")
+
+        for compound_data in compounds_data:
+            compound_uuid = self._get_or_generate_uuid(compound_data['uuid'])
+
+            compound = Compound.objects.create(
+                uuid=compound_uuid,
+                package=package,
+                name=compound_data['name'],
+                description=compound_data['description'],
+                kv=compound_data.get('kv', {}),
+                # default_structure will be set later
+            )
+
+            # Set aliases if present
+            if compound_data.get('aliases'):
+                compound.aliases = compound_data['aliases']
+                compound.save()
+
+            self._cache_object('Compound', compound_data['uuid'], compound)
+
+            # Handle external identifiers
+            self._create_external_identifiers(compound, compound_data.get('external_identifiers', []))
+
+    def _import_structures(self, package: Package, structures_data: List[Dict[str, Any]]):
+        """Import compound structures."""
+        print(f"Importing {len(structures_data)} compound structures...")
+
+        for structure_data in structures_data:
+            structure_uuid = self._get_or_generate_uuid(structure_data['uuid'])
+            compound_uuid = structure_data['compound']['uuid']
+            compound = self._get_cached_object('Compound', compound_uuid)
+
+            if not compound:
+                print(f"Warning: Compound with UUID {compound_uuid} not found for structure")
+                continue
+
+            structure = CompoundStructure.objects.create(
+                uuid=structure_uuid,
+                compound=compound,
+                name=structure_data['name'],
+                description=structure_data['description'],
+                kv=structure_data.get('kv', {}),
+                smiles=structure_data['smiles'],
+                canonical_smiles=structure_data['canonical_smiles'],
+                inchikey=structure_data['inchikey'],
+                normalized_structure=structure_data.get('normalized_structure', False)
+            )
+
+            # Set aliases if present
+            if structure_data.get('aliases'):
+                structure.aliases = structure_data['aliases']
+                structure.save()
+
+            self._cache_object('CompoundStructure', structure_data['uuid'], structure)
+
+            # Handle external identifiers
+            self._create_external_identifiers(structure, structure_data.get('external_identifiers', []))
+
+    def _import_rules(self, package: Package, rules_data: Dict[str, Any]):
+        """Import all types of rules."""
+        print("Importing rules...")
+
+        # Import simple rules first
+        simple_rules_data = rules_data.get('simple_rules', [])
+        print(f"Importing {len(simple_rules_data)} simple rules...")
+
+        for rule_data in simple_rules_data:
+            self._create_simple_rule(package, rule_data)
+
+        # Import parallel rules
+        parallel_rules_data = rules_data.get('parallel_rules', [])
+        print(f"Importing {len(parallel_rules_data)} parallel rules...")
+
+        for rule_data in parallel_rules_data:
+            self._create_parallel_rule(package, rule_data)
+
+    def _create_simple_rule(self, package: Package, rule_data: Dict[str, Any]):
+        """Create a simple rule (SimpleAmbitRule or SimpleRDKitRule)."""
+        rule_uuid = self._get_or_generate_uuid(rule_data['uuid'])
+        rule_type = rule_data.get('rule_type', 'SimpleRule')
+
+        common_fields = {
+            'uuid': rule_uuid,
+            'package': package,
+            'name': rule_data['name'],
+            'description': rule_data['description'],
+            'kv': rule_data.get('kv', {})
+        }
+
+        if rule_type == 'SimpleAmbitRule':
+            rule = SimpleAmbitRule.objects.create(
+                **common_fields,
+                smirks=rule_data.get('smirks', ''),
+                reactant_filter_smarts=rule_data.get('reactant_filter_smarts', ''),
+                product_filter_smarts=rule_data.get('product_filter_smarts', '')
+            )
+        elif rule_type == 'SimpleRDKitRule':
+            rule = SimpleRDKitRule.objects.create(
+                **common_fields,
+                reaction_smarts=rule_data.get('reaction_smarts', '')
+            )
+        else:
+            rule = SimpleRule.objects.create(**common_fields)
+
+        # Set aliases if present
+        if rule_data.get('aliases'):
+            rule.aliases = rule_data['aliases']
+            rule.save()
+
+        self._cache_object('SimpleRule', rule_data['uuid'], rule)
+        return rule
+
+    def _create_parallel_rule(self, package: Package, rule_data: Dict[str, Any]):
+        """Create a parallel rule."""
+        rule_uuid = self._get_or_generate_uuid(rule_data['uuid'])
+
+        rule = ParallelRule.objects.create(
+            uuid=rule_uuid,
+            package=package,
+            name=rule_data['name'],
+            description=rule_data['description'],
+            kv=rule_data.get('kv', {})
+        )
+
+        # Set aliases if present
+        if rule_data.get('aliases'):
+            rule.aliases = rule_data['aliases']
+            rule.save()
+
+        # Add simple rules
+        for simple_rule_ref in rule_data.get('simple_rules', []):
+            simple_rule = self._get_cached_object('SimpleRule', simple_rule_ref['uuid'])
+            if simple_rule:
+                rule.simple_rules.add(simple_rule)
+
+        self._cache_object('ParallelRule', rule_data['uuid'], rule)
+        return rule
+
+    def _import_reactions(self, package: Package, reactions_data: List[Dict[str, Any]]):
+        """Import reactions."""
+        print(f"Importing {len(reactions_data)} reactions...")
+
+        for reaction_data in reactions_data:
+            reaction_uuid = self._get_or_generate_uuid(reaction_data['uuid'])
+
+            reaction = Reaction.objects.create(
+                uuid=reaction_uuid,
+                package=package,
+                name=reaction_data['name'],
+                description=reaction_data['description'],
+                kv=reaction_data.get('kv', {}),
+                multi_step=reaction_data.get('multi_step', False),
+                medline_references=reaction_data.get('medline_references', [])
+            )
+
+            # Set aliases if present
+            if reaction_data.get('aliases'):
+                reaction.aliases = reaction_data['aliases']
+                reaction.save()
+
+            # Add educts and products
+            for educt_ref in reaction_data.get('educts', []):
+                compound = self._get_cached_object('CompoundStructure', educt_ref['uuid'])
+                if compound:
+                    reaction.educts.add(compound)
+
+            for product_ref in reaction_data.get('products', []):
+                compound = self._get_cached_object('CompoundStructure', product_ref['uuid'])
+                if compound:
+                    reaction.products.add(compound)
+
+            # Add rules
+            for rule_ref in reaction_data.get('rules', []):
+                # Try to find rule in different caches
+                rule = (self._get_cached_object('SimpleRule', rule_ref['uuid']) or
+                        self._get_cached_object('ParallelRule', rule_ref['uuid']))
+                if rule:
+                    reaction.rules.add(rule)
+
+            self._cache_object('Reaction', reaction_data['uuid'], reaction)
+
+            # Handle external identifiers
+            self._create_external_identifiers(reaction, reaction_data.get('external_identifiers', []))
+
+    def _import_pathways(self, package: Package, pathways_data: List[Dict[str, Any]]):
+        """Import pathways."""
+        print(f"Importing {len(pathways_data)} pathways...")
+
+        for pathway_data in pathways_data:
+            pathway_uuid = self._get_or_generate_uuid(pathway_data['uuid'])
+
+            pathway = Pathway.objects.create(
+                uuid=pathway_uuid,
+                package=package,
+                name=pathway_data['name'],
+                description=pathway_data['description'],
+                kv=pathway_data.get('kv', {})
+                # setting will be handled separately if needed
+            )
+
+            # Set aliases if present
+            if pathway_data.get('aliases'):
+                pathway.aliases = pathway_data['aliases']
+                pathway.save()
+
+            self._cache_object('Pathway', pathway_data['uuid'], pathway)
+
+    def _import_nodes(self, package: Package, nodes_data: List[Dict[str, Any]]):
+        """Import pathway nodes."""
+        print(f"Importing {len(nodes_data)} nodes...")
+
+        for node_data in nodes_data:
+            node_uuid = self._get_or_generate_uuid(node_data['uuid'])
+            pathway_uuid = node_data['pathway']['uuid']
+            pathway = self._get_cached_object('Pathway', pathway_uuid)
+
+            if not pathway:
+                print(f"Warning: Pathway with UUID {pathway_uuid} not found for node")
+                continue
+
+            # For now, we'll set default_node_label to None and handle it later
+            # as it requires compound structures to be fully imported
+            node = Node.objects.create(
+                uuid=node_uuid,
+                pathway=pathway,
+                name=node_data['name'],
+                description=node_data['description'],
+                kv=node_data.get('kv', {}),
+                depth=node_data.get('depth', 0),
+                default_node_label=self._get_cached_object('CompoundStructure', node_data['default_node_label']['uuid'])
+            )
+
+            # Set aliases if present
+            if node_data.get('aliases'):
+                node.aliases = node_data['aliases']
+                node.save()
+
+            self._cache_object('Node', node_data['uuid'], node)
+            # Store node_data for later processing of relationships
+            node._import_data = node_data
+
+    def _import_edges(self, package: Package, edges_data: List[Dict[str, Any]]):
+        """Import pathway edges."""
+        print(f"Importing {len(edges_data)} edges...")
+
+        for edge_data in edges_data:
+            edge_uuid = self._get_or_generate_uuid(edge_data['uuid'])
+            pathway_uuid = edge_data['pathway']['uuid']
+            pathway = self._get_cached_object('Pathway', pathway_uuid)
+
+            if not pathway:
+                print(f"Warning: Pathway with UUID {pathway_uuid} not found for edge")
+                continue
+
+            # For now, we'll set edge_label to None and handle it later
+            edge = Edge.objects.create(
+                uuid=edge_uuid,
+                pathway=pathway,
+                name=edge_data['name'],
+                description=edge_data['description'],
+                kv=edge_data.get('kv', {}),
+                edge_label=None  # Will be set later
+            )
+
+            # Set aliases if present
+            if edge_data.get('aliases'):
+                edge.aliases = edge_data['aliases']
+                edge.save()
+
+            # Add start and end nodes
+            for start_node_ref in edge_data.get('start_nodes', []):
+                node = self._get_cached_object('Node', start_node_ref['uuid'])
+                if node:
+                    edge.start_nodes.add(node)
+
+            for end_node_ref in edge_data.get('end_nodes', []):
+                node = self._get_cached_object('Node', end_node_ref['uuid'])
+                if node:
+                    edge.end_nodes.add(node)
+
+            self._cache_object('Edge', edge_data['uuid'], edge)
+
+    def _import_scenarios(self, package: Package, scenarios_data: List[Dict[str, Any]]):
+        """Import scenarios."""
+        print(f"Importing {len(scenarios_data)} scenarios...")
+
+        # First pass: create scenarios without parent relationships
+        for scenario_data in scenarios_data:
+            scenario_uuid = self._get_or_generate_uuid(scenario_data['uuid'])
+
+            scenario_date = None
+            if scenario_data.get('scenario_date'):
+                scenario_date = scenario_data['scenario_date']
+
+            scenario = Scenario.objects.create(
+                uuid=scenario_uuid,
+                package=package,
+                name=scenario_data['name'],
+                description=scenario_data['description'],
+                kv=scenario_data.get('kv', {}),
+                scenario_date=scenario_date,
+                scenario_type=scenario_data.get('scenario_type'),
+                additional_information=scenario_data.get('additional_information', {})
+            )
+
+            self._cache_object('Scenario', scenario_data['uuid'], scenario)
+            # Store scenario_data for later processing of parent relationships
+            scenario._import_data = scenario_data
+
+        # Second pass: set parent relationships
+        for scenario_data in scenarios_data:
+            if scenario_data.get('parent'):
+                scenario = self._get_cached_object('Scenario', scenario_data['uuid'])
+                parent = self._get_cached_object('Scenario', scenario_data['parent']['uuid'])
+                if scenario and parent:
+                    scenario.parent = parent
+                    scenario.save()
+
+    def _import_models(self, package: Package, models_data: List[Dict[str, Any]]):
+        """Import EPModels."""
+        print(f"Importing {len(models_data)} models...")
+
+        for model_data in models_data:
+            model_uuid = self._get_or_generate_uuid(model_data['uuid'])
+            model_type = model_data.get('model_type', 'EPModel')
+
+            common_fields = {
+                'uuid': model_uuid,
+                'package': package,
+                'name': model_data['name'],
+                'description': model_data['description'],
+                'kv': model_data.get('kv', {})
+            }
+
+            # Add PackageBasedModel fields if present
+            if 'threshold' in model_data:
+                common_fields.update({
+                    'threshold': model_data.get('threshold'),
+                    'eval_results': model_data.get('eval_results', {}),
+                    'model_status': model_data.get('model_status', 'INITIAL')
+                })
+
+            # Create the appropriate model type
+            if model_type == 'RuleBasedRelativeReasoning':
+                model = RuleBasedRelativeReasoning.objects.create(
+                    **common_fields,
+                    min_count=model_data.get('min_count', 1),
+                    max_count=model_data.get('max_count', 10)
+                )
+            elif model_type == 'MLRelativeReasoning':
+                model = MLRelativeReasoning.objects.create(**common_fields)
+            elif model_type == 'EnviFormer':
+                model = EnviFormer.objects.create(**common_fields)
+            elif model_type == 'PluginModel':
+                model = PluginModel.objects.create(**common_fields)
+            else:
+                model = EPModel.objects.create(**common_fields)
+
+            # Set aliases if present
+            if model_data.get('aliases'):
+                model.aliases = model_data['aliases']
+                model.save()
+
+            # Add package relationships for PackageBasedModel
+            if hasattr(model, 'rule_packages'):
+                for pkg_ref in model_data.get('rule_packages', []):
+                    pkg = self._get_cached_object('Package', pkg_ref['uuid'])
+                    if pkg:
+                        model.rule_packages.add(pkg)
+
+                for pkg_ref in model_data.get('data_packages', []):
+                    pkg = self._get_cached_object('Package', pkg_ref['uuid'])
+                    if pkg:
+                        model.data_packages.add(pkg)
+
+                for pkg_ref in model_data.get('eval_packages', []):
+                    pkg = self._get_cached_object('Package', pkg_ref['uuid'])
+                    if pkg:
+                        model.eval_packages.add(pkg)
+
+            self._cache_object('EPModel', model_data['uuid'], model)
+
+    def _set_default_structures(self, compounds_data: List[Dict[str, Any]]):
+        """Set default structures for compounds after all structures are created."""
+        print("Setting default structures for compounds...")
+
+        for compound_data in compounds_data:
+            if compound_data.get('default_structure'):
+                compound = self._get_cached_object('Compound', compound_data['uuid'])
+                structure = self._get_cached_object('CompoundStructure',
+                                                    compound_data['default_structure']['uuid'])
+                if compound and structure:
+                    compound.default_structure = structure
+                    compound.save()
+
+    def _create_external_identifiers(self, obj, identifiers_data: List[Dict[str, Any]]):
+        """Create external identifiers for an object."""
+        for identifier_data in identifiers_data:
+            # Get or create the external database
+            db_data = identifier_data['database']
+            database, _ = ExternalDatabase.objects.get_or_create(
+                name=db_data['name'],
+                defaults={
+                    'base_url': db_data.get('base_url', ''),
+                    'full_name': db_data.get('name', ''),
+                    'description': '',
+                    'is_active': True
+                }
+            )
+
+            # Create the external identifier
+            ExternalIdentifier.objects.create(
+                content_object=obj,
+                database=database,
+                identifier_value=identifier_data['identifier_value'],
+                url=identifier_data.get('url', ''),
+                is_primary=identifier_data.get('is_primary', False)
+            )

uv.lock

@@ -501,6 +501,35 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/30/66/00e6fc8ca9cf10ca41aa5d3d7746214fa7690138174c6716e9a08259bff2/django_polymorphic-4.1.0-py3-none-any.whl", hash = "sha256:0ce3984999e103a0d1a434a5c5617f2c7f990dc3d5fb3585ce0fadadf9ff90ea", size = 62845 },
 ]
 
+[[package]]
+name = "django-stubs"
+version = "5.2.4"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "django" },
+    { name = "django-stubs-ext" },
+    { name = "tomli", marker = "python_full_version < '3.11'" },
+    { name = "types-pyyaml" },
+    { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/fd/71/430901a1a799e1e66bb169a28e48a5f7b6e4ad5d6cadf60d9319d44b4181/django_stubs-5.2.4.tar.gz", hash = "sha256:89e1493d57ed091fdc990bf625bde77088e8660fcf6a3cdf0faf68e33c11c79b", size = 247751 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/e0/0a/6e1326888d2a8b5f390a28b543a9766105bff573641bbfd7edc7394ab5fe/django_stubs-5.2.4-py3-none-any.whl", hash = "sha256:000d59e826a15b032070d4b1b7f95b281005d690493173b95837c4c07b68dcf6", size = 490115 },
+]
+
+[[package]]
+name = "django-stubs-ext"
+version = "5.2.4"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "django" },
+    { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/63/9a/d3554928fca79bc8e28dccb7f9c410c132667a47a216b53850c268a94114/django_stubs_ext-5.2.4.tar.gz", hash = "sha256:920bb81c1fbf95b8882db7dcd3fff2d851a3c01d49aef9c7b5d79da40ff39fe6", size = 6488 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/59/20/4595270c02c5763ff751dab83df148f606de3738a236a4c961c71df7c005/django_stubs_ext-5.2.4-py3-none-any.whl", hash = "sha256:e539cb40c5281f291c285ba7ecd704e70343d8f610f13629c2b1521454364d1b", size = 9307 },
+]
+
 [[package]]
 name = "enviformer"
 version = "0.1.0"
@@ -522,12 +551,12 @@ dependencies = [
     { name = "django-ninja" },
     { name = "django-oauth-toolkit" },
     { name = "django-polymorphic" },
+    { name = "django-stubs" },
     { name = "enviformer" },
     { name = "envipy-additional-information" },
     { name = "envipy-plugins" },
     { name = "epam-indigo" },
     { name = "gunicorn" },
-    { name = "msal" },
     { name = "psycopg2-binary" },
     { name = "python-dotenv" },
     { name = "rdkit" },
@@ -538,6 +567,11 @@ dependencies = [
     { name = "setuptools" },
 ]
 
+[package.optional-dependencies]
+ms-login = [
+    { name = "msal" },
+]
+
 [package.metadata]
 requires-dist = [
     { name = "celery", specifier = ">=5.5.2" },
@@ -547,12 +581,13 @@ requires-dist = [
     { name = "django-ninja", specifier = ">=1.4.1" },
     { name = "django-oauth-toolkit", specifier = ">=3.0.1" },
     { name = "django-polymorphic", specifier = ">=4.1.0" },
+    { name = "django-stubs", specifier = ">=5.2.4" },
     { name = "enviformer", git = "ssh://git@git.envipath.com/enviPath/enviformer.git?rev=v0.1.0" },
     { name = "envipy-additional-information", git = "ssh://git@git.envipath.com/enviPath/enviPy-additional-information.git?rev=v0.1.4" },
     { name = "envipy-plugins", git = "ssh://git@git.envipath.com/enviPath/enviPy-plugins.git?rev=v0.1.0" },
     { name = "epam-indigo", specifier = ">=1.30.1" },
     { name = "gunicorn", specifier = ">=23.0.0" },
-    { name = "msal", specifier = ">=1.33.0" },
+    { name = "msal", marker = "extra == 'ms-login'", specifier = ">=1.33.0" },
     { name = "psycopg2-binary", specifier = ">=2.9.10" },
     { name = "python-dotenv", specifier = ">=1.1.0" },
     { name = "rdkit", specifier = ">=2025.3.2" },
@@ -1870,6 +1905,45 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/32/d5/f9a850d79b0851d1d4ef6456097579a9005b31fea68726a4ae5f2d82ddd9/threadpoolctl-3.6.0-py3-none-any.whl", hash = "sha256:43a0b8fd5a2928500110039e43a5eed8480b918967083ea48dc3ab9f13c4a7fb", size = 18638 },
 ]
 
+[[package]]
+name = "tomli"
+version = "2.2.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/18/87/302344fed471e44a87289cf4967697d07e532f2421fdaf868a303cbae4ff/tomli-2.2.1.tar.gz", hash = "sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff", size = 17175 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/43/ca/75707e6efa2b37c77dadb324ae7d9571cb424e61ea73fad7c56c2d14527f/tomli-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249", size = 131077 },
+    { url = "https://files.pythonhosted.org/packages/c7/16/51ae563a8615d472fdbffc43a3f3d46588c264ac4f024f63f01283becfbb/tomli-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6", size = 123429 },
+    { url = "https://files.pythonhosted.org/packages/f1/dd/4f6cd1e7b160041db83c694abc78e100473c15d54620083dbd5aae7b990e/tomli-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a", size = 226067 },
+    { url = "https://files.pythonhosted.org/packages/a9/6b/c54ede5dc70d648cc6361eaf429304b02f2871a345bbdd51e993d6cdf550/tomli-2.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6972ca9c9cc9f0acaa56a8ca1ff51e7af152a9f87fb64623e31d5c83700080ee", size = 236030 },
+    { url = "https://files.pythonhosted.org/packages/1f/47/999514fa49cfaf7a92c805a86c3c43f4215621855d151b61c602abb38091/tomli-2.2.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c954d2250168d28797dd4e3ac5cf812a406cd5a92674ee4c8f123c889786aa8e", size = 240898 },
+    { url = "https://files.pythonhosted.org/packages/73/41/0a01279a7ae09ee1573b423318e7934674ce06eb33f50936655071d81a24/tomli-2.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8dd28b3e155b80f4d54beb40a441d366adcfe740969820caf156c019fb5c7ec4", size = 229894 },
+    { url = "https://files.pythonhosted.org/packages/55/18/5d8bc5b0a0362311ce4d18830a5d28943667599a60d20118074ea1b01bb7/tomli-2.2.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:e59e304978767a54663af13c07b3d1af22ddee3bb2fb0618ca1593e4f593a106", size = 245319 },
+    { url = "https://files.pythonhosted.org/packages/92/a3/7ade0576d17f3cdf5ff44d61390d4b3febb8a9fc2b480c75c47ea048c646/tomli-2.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:33580bccab0338d00994d7f16f4c4ec25b776af3ffaac1ed74e0b3fc95e885a8", size = 238273 },
+    { url = "https://files.pythonhosted.org/packages/72/6f/fa64ef058ac1446a1e51110c375339b3ec6be245af9d14c87c4a6412dd32/tomli-2.2.1-cp311-cp311-win32.whl", hash = "sha256:465af0e0875402f1d226519c9904f37254b3045fc5084697cefb9bdde1ff99ff", size = 98310 },
+    { url = "https://files.pythonhosted.org/packages/6a/1c/4a2dcde4a51b81be3530565e92eda625d94dafb46dbeb15069df4caffc34/tomli-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:2d0f2fdd22b02c6d81637a3c95f8cd77f995846af7414c5c4b8d0545afa1bc4b", size = 108309 },
+    { url = "https://files.pythonhosted.org/packages/52/e1/f8af4c2fcde17500422858155aeb0d7e93477a0d59a98e56cbfe75070fd0/tomli-2.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4a8f6e44de52d5e6c657c9fe83b562f5f4256d8ebbfe4ff922c495620a7f6cea", size = 132762 },
+    { url = "https://files.pythonhosted.org/packages/03/b8/152c68bb84fc00396b83e7bbddd5ec0bd3dd409db4195e2a9b3e398ad2e3/tomli-2.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8d57ca8095a641b8237d5b079147646153d22552f1c637fd3ba7f4b0b29167a8", size = 123453 },
+    { url = "https://files.pythonhosted.org/packages/c8/d6/fc9267af9166f79ac528ff7e8c55c8181ded34eb4b0e93daa767b8841573/tomli-2.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e340144ad7ae1533cb897d406382b4b6fede8890a03738ff1683af800d54192", size = 233486 },
+    { url = "https://files.pythonhosted.org/packages/5c/51/51c3f2884d7bab89af25f678447ea7d297b53b5a3b5730a7cb2ef6069f07/tomli-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db2b95f9de79181805df90bedc5a5ab4c165e6ec3fe99f970d0e302f384ad222", size = 242349 },
+    { url = "https://files.pythonhosted.org/packages/ab/df/bfa89627d13a5cc22402e441e8a931ef2108403db390ff3345c05253935e/tomli-2.2.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40741994320b232529c802f8bc86da4e1aa9f413db394617b9a256ae0f9a7f77", size = 252159 },
+    { url = "https://files.pythonhosted.org/packages/9e/6e/fa2b916dced65763a5168c6ccb91066f7639bdc88b48adda990db10c8c0b/tomli-2.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:400e720fe168c0f8521520190686ef8ef033fb19fc493da09779e592861b78c6", size = 237243 },
+    { url = "https://files.pythonhosted.org/packages/b4/04/885d3b1f650e1153cbb93a6a9782c58a972b94ea4483ae4ac5cedd5e4a09/tomli-2.2.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:02abe224de6ae62c19f090f68da4e27b10af2b93213d36cf44e6e1c5abd19fdd", size = 259645 },
+    { url = "https://files.pythonhosted.org/packages/9c/de/6b432d66e986e501586da298e28ebeefd3edc2c780f3ad73d22566034239/tomli-2.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b82ebccc8c8a36f2094e969560a1b836758481f3dc360ce9a3277c65f374285e", size = 244584 },
+    { url = "https://files.pythonhosted.org/packages/1c/9a/47c0449b98e6e7d1be6cbac02f93dd79003234ddc4aaab6ba07a9a7482e2/tomli-2.2.1-cp312-cp312-win32.whl", hash = "sha256:889f80ef92701b9dbb224e49ec87c645ce5df3fa2cc548664eb8a25e03127a98", size = 98875 },
+    { url = "https://files.pythonhosted.org/packages/ef/60/9b9638f081c6f1261e2688bd487625cd1e660d0a85bd469e91d8db969734/tomli-2.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:7fc04e92e1d624a4a63c76474610238576942d6b8950a2d7f908a340494e67e4", size = 109418 },
+    { url = "https://files.pythonhosted.org/packages/04/90/2ee5f2e0362cb8a0b6499dc44f4d7d48f8fff06d28ba46e6f1eaa61a1388/tomli-2.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7", size = 132708 },
+    { url = "https://files.pythonhosted.org/packages/c0/ec/46b4108816de6b385141f082ba99e315501ccd0a2ea23db4a100dd3990ea/tomli-2.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:286f0ca2ffeeb5b9bd4fcc8d6c330534323ec51b2f52da063b11c502da16f30c", size = 123582 },
+    { url = "https://files.pythonhosted.org/packages/a0/bd/b470466d0137b37b68d24556c38a0cc819e8febe392d5b199dcd7f578365/tomli-2.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a92ef1a44547e894e2a17d24e7557a5e85a9e1d0048b0b5e7541f76c5032cb13", size = 232543 },
+    { url = "https://files.pythonhosted.org/packages/d9/e5/82e80ff3b751373f7cead2815bcbe2d51c895b3c990686741a8e56ec42ab/tomli-2.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9316dc65bed1684c9a98ee68759ceaed29d229e985297003e494aa825ebb0281", size = 241691 },
+    { url = "https://files.pythonhosted.org/packages/05/7e/2a110bc2713557d6a1bfb06af23dd01e7dde52b6ee7dadc589868f9abfac/tomli-2.2.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e85e99945e688e32d5a35c1ff38ed0b3f41f43fad8df0bdf79f72b2ba7bc5272", size = 251170 },
+    { url = "https://files.pythonhosted.org/packages/64/7b/22d713946efe00e0adbcdfd6d1aa119ae03fd0b60ebed51ebb3fa9f5a2e5/tomli-2.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ac065718db92ca818f8d6141b5f66369833d4a80a9d74435a268c52bdfa73140", size = 236530 },
+    { url = "https://files.pythonhosted.org/packages/38/31/3a76f67da4b0cf37b742ca76beaf819dca0ebef26d78fc794a576e08accf/tomli-2.2.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:d920f33822747519673ee656a4b6ac33e382eca9d331c87770faa3eef562aeb2", size = 258666 },
+    { url = "https://files.pythonhosted.org/packages/07/10/5af1293da642aded87e8a988753945d0cf7e00a9452d3911dd3bb354c9e2/tomli-2.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a198f10c4d1b1375d7687bc25294306e551bf1abfa4eace6650070a5c1ae2744", size = 243954 },
+    { url = "https://files.pythonhosted.org/packages/5b/b9/1ed31d167be802da0fc95020d04cd27b7d7065cc6fbefdd2f9186f60d7bd/tomli-2.2.1-cp313-cp313-win32.whl", hash = "sha256:d3f5614314d758649ab2ab3a62d4f2004c825922f9e370b29416484086b264ec", size = 98724 },
+    { url = "https://files.pythonhosted.org/packages/c7/32/b0963458706accd9afcfeb867c0f9175a741bf7b19cd424230714d722198/tomli-2.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:a38aa0308e754b0e3c67e344754dff64999ff9b513e691d0e786265c93583c69", size = 109383 },
+    { url = "https://files.pythonhosted.org/packages/6e/c2/61d3e0f47e2b74ef40a68b9e6ad5984f6241a942f7cd3bbfbdbd03861ea9/tomli-2.2.1-py3-none-any.whl", hash = "sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc", size = 14257 },
+]
+
 [[package]]
 name = "torch"
 version = "2.7.0"
@@ -1963,6 +2037,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/0a/93/f28a696fa750b9b608baa236f8225dd3290e5aff27433b06143adc025961/triton-3.3.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ce4700fc14032af1e049005ae94ba908e71cd6c2df682239aed08e49bc71b742", size = 156580729 },
 ]
 
+[[package]]
+name = "types-pyyaml"
+version = "6.0.12.20250822"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/49/85/90a442e538359ab5c9e30de415006fb22567aa4301c908c09f19e42975c2/types_pyyaml-6.0.12.20250822.tar.gz", hash = "sha256:259f1d93079d335730a9db7cff2bcaf65d7e04b4a56b5927d49a612199b59413", size = 17481 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/32/8e/8f0aca667c97c0d76024b37cffa39e76e2ce39ca54a38f285a64e6ae33ba/types_pyyaml-6.0.12.20250822-py3-none-any.whl", hash = "sha256:1fe1a5e146aa315483592d292b72a172b65b946a6d98aa6ddd8e4aa838ab7098", size = 20314 },
+]
+
 [[package]]
 name = "typing-extensions"
 version = "4.13.2"