From 97122d1d743ad2dcf5f6ad6b88deee66a234ef6d Mon Sep 17 00:00:00 2001 From: Jakub Komarek Date: Wed, 2 Jul 2025 23:29:26 +0200 Subject: [PATCH 001/472] CommunityLibrarySubmission model --- .../0154_community_library_submission.py | 85 +++++++++++++++++++ contentcuration/contentcuration/models.py | 79 +++++++++++++++++ 2 files changed, 164 insertions(+) create mode 100644 contentcuration/contentcuration/migrations/0154_community_library_submission.py diff --git a/contentcuration/contentcuration/migrations/0154_community_library_submission.py b/contentcuration/contentcuration/migrations/0154_community_library_submission.py new file mode 100644 index 0000000000..e70df7e82b --- /dev/null +++ b/contentcuration/contentcuration/migrations/0154_community_library_submission.py @@ -0,0 +1,85 @@ +# Generated by Django 3.2.24 on 2025-07-02 20:48 +import django.db.models.deletion +from django.conf import settings +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + + dependencies = [ + ("contentcuration", "0153_alter_recommendationsevent_time_hidden"), + ] + + operations = [ + migrations.CreateModel( + name="Country", + fields=[ + ( + "code", + models.CharField( + help_text="alpha-2 country code", + max_length=2, + primary_key=True, + serialize=False, + ), + ), + ("name", models.CharField(max_length=100, unique=True)), + ], + ), + migrations.CreateModel( + name="CommunityLibrarySubmission", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("description", models.TextField(blank=True, max_length=400)), + ("channel_version", models.IntegerField()), + ("categories", models.JSONField(blank=True, default=list)), + ("date_created", models.DateTimeField(auto_now_add=True)), + ( + "status", + models.CharField( + choices=[ + ("P", "Pending"), + ("F", "Flagged for review"), + ("A", "Approved"), + ("L", "Live"), + ], + default="P", + max_length=1, + ), + ), + ( + "author", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="community_library_submissions", + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "channel", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="community_library_submissions", + to="contentcuration.channel", + ), + ), + ("countries", models.ManyToManyField(to="contentcuration.Country")), + ], + ), + migrations.AddConstraint( + model_name="communitylibrarysubmission", + constraint=models.UniqueConstraint( + fields=("channel", "channel_version"), + name="unique_channel_with_channel_version", + ), + ), + ] diff --git a/contentcuration/contentcuration/models.py b/contentcuration/contentcuration/models.py index f193921afb..ee07197869 100644 --- a/contentcuration/contentcuration/models.py +++ b/contentcuration/contentcuration/models.py @@ -2532,6 +2532,85 @@ def __str__(self): return self.ietf_name() +class Country(models.Model): + code = models.CharField( + max_length=2, primary_key=True, help_text="alpha-2 country code" + ) + name = models.CharField(max_length=100, unique=True) + + +class CommunityLibrarySubmission(models.Model): + class Status(models.TextChoices): + PENDING = "P", _("Pending") + FLAGGED = "F", _("Flagged for review") + APPROVED = "A", _("Approved") + LIVE = "L", _("Live") + + description = models.TextField(max_length=400, blank=True) + channel = models.ForeignKey( + Channel, + related_name="community_library_submissions", + on_delete=models.CASCADE, + ) + channel_version = models.IntegerField() + author = models.ForeignKey( + User, + related_name="community_library_submissions", + on_delete=models.CASCADE, + ) + countries = models.ManyToManyField(Country) + categories = models.JSONField(default=list, blank=True) + date_created = models.DateTimeField(auto_now_add=True) + status = models.CharField( + max_length=1, choices=Status.choices, default=Status.PENDING + ) + + def clean(self): + """ + Validate that the submission author is an editor of the channel. + This cannot be expressed as a constraint because traversing + related fields is not supported in constraints. Note that this + method is NOT called automatically on every save. + """ + super().clean() + if not self.channel.editors.filter(pk=self.author.pk).exists(): + raise ValidationError( + _( + "The submission author must be an editor of the channel the submission " + "belong to" + ), + code="author_not_editor", + ) + + @classmethod + def filter_view_queryset(cls, queryset, user): + if user.is_anonymous: + return queryset.none() + + if user.is_admin: + return queryset + + return queryset.filter(channel__editors=user) + + @classmethod + def filter_edit_queryset(cls, queryset, user): + if user.is_anonymous: + return queryset.none() + + if user.is_admin: + return queryset + + return queryset.filter(author=user, channel__editors=user) + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["channel", "channel_version"], + name="unique_channel_with_channel_version", + ), + ] + + ASSESSMENT_ID_INDEX_NAME = "assessment_id_idx" From 6a3c5c4089576e4af6aed67cdb270697f4f2a3a7 Mon Sep 17 00:00:00 2001 From: Jakub Komarek Date: Wed, 2 Jul 2025 23:32:36 +0200 Subject: [PATCH 002/472] CommunityLibrarySubmission tests --- .../contentcuration/tests/test_models.py | 125 ++++++++++++++++++ .../contentcuration/tests/testdata.py | 19 +++ 2 files changed, 144 insertions(+) diff --git a/contentcuration/contentcuration/tests/test_models.py b/contentcuration/contentcuration/tests/test_models.py index 2fb728e4a3..aecf725c64 100644 --- a/contentcuration/contentcuration/tests/test_models.py +++ b/contentcuration/contentcuration/tests/test_models.py @@ -18,6 +18,7 @@ from contentcuration.models import Channel from contentcuration.models import ChannelHistory from contentcuration.models import ChannelSet +from contentcuration.models import CommunityLibrarySubmission from contentcuration.models import ContentNode from contentcuration.models import CONTENTNODE_TREE_ID_CACHE_KEY from contentcuration.models import File @@ -547,6 +548,130 @@ def test_make_content_id_unique(self): self.assertNotEqual(copied_node_old_content_id, copied_node.content_id) +class CommunityLibrarySubmissionTestCase(PermissionQuerysetTestCase): + @property + def base_queryset(self): + return CommunityLibrarySubmission.objects.all() + + def test_create_submission(self): + # Smoke test + channel = testdata.channel() + author = testdata.user() + channel.editors.add(author) + channel.save() + + country = testdata.country() + + submission = CommunityLibrarySubmission.objects.create( + description="Test submission", + channel=channel, + channel_version=1, + author=author, + categories=["test_category"], + status=CommunityLibrarySubmission.Status.PENDING, + ) + submission.countries.add(country) + + submission.full_clean() + submission.save() + + def test_filter_view_queryset__anonymous(self): + _ = testdata.community_library_submission() + + queryset = CommunityLibrarySubmission.filter_view_queryset( + self.base_queryset, user=self.anonymous_user + ) + self.assertFalse(queryset.exists()) + + def test_filter_view_queryset__forbidden_user(self): + _ = testdata.community_library_submission() + + queryset = CommunityLibrarySubmission.filter_view_queryset( + self.base_queryset, user=self.forbidden_user + ) + self.assertFalse(queryset.exists()) + + def test_filter_view_queryset__channel_editor(self): + submission_a = testdata.community_library_submission() + submission_b = testdata.community_library_submission() + + user = testdata.user() + submission_a.channel.editors.add(user) + submission_a.save() + + queryset = CommunityLibrarySubmission.filter_view_queryset( + self.base_queryset, user=user + ) + self.assertQuerysetContains(queryset, pk=submission_a.id) + self.assertQuerysetDoesNotContain(queryset, pk=submission_b.id) + + def test_filter_view_queryset__admin(self): + submission_a = testdata.community_library_submission() + + queryset = CommunityLibrarySubmission.filter_view_queryset( + self.base_queryset, user=self.admin_user + ) + self.assertQuerysetContains(queryset, pk=submission_a.id) + + def test_filter_edit_queryset__anonymous(self): + _ = testdata.community_library_submission() + + queryset = CommunityLibrarySubmission.filter_edit_queryset( + self.base_queryset, user=self.anonymous_user + ) + self.assertFalse(queryset.exists()) + + def test_filter_edit_queryset__forbidden_user(self): + _ = testdata.community_library_submission() + + queryset = CommunityLibrarySubmission.filter_edit_queryset( + self.base_queryset, user=self.forbidden_user + ) + self.assertFalse(queryset.exists()) + + def test_filter_edit_queryset__channel_editor(self): + submission = testdata.community_library_submission() + + user = testdata.user() + submission.channel.editors.add(user) + submission.save() + + queryset = CommunityLibrarySubmission.filter_edit_queryset( + self.base_queryset, user=user + ) + self.assertFalse(queryset.exists()) + + def test_filter_edit_queryset__author__channel_editor(self): + submission_a = testdata.community_library_submission() + submission_b = testdata.community_library_submission() + + queryset = CommunityLibrarySubmission.filter_edit_queryset( + self.base_queryset, user=submission_a.author + ) + self.assertQuerysetContains(queryset, pk=submission_a.id) + self.assertQuerysetDoesNotContain(queryset, pk=submission_b.id) + + def test_filter_edit_queryset__author__not_channel_editor(self): + submission = testdata.community_library_submission() + + user = testdata.user(email="new@user.com") + submission.author = user + submission.save() + + queryset = CommunityLibrarySubmission.filter_edit_queryset( + self.base_queryset, user=user + ) + self.assertFalse(queryset.exists()) + + def test_filter_edit_queryset__admin(self): + submission_a = testdata.community_library_submission() + + queryset = CommunityLibrarySubmission.filter_edit_queryset( + self.base_queryset, user=self.admin_user + ) + self.assertQuerysetContains(queryset, pk=submission_a.id) + + class AssessmentItemTestCase(PermissionQuerysetTestCase): @property def base_queryset(self): diff --git a/contentcuration/contentcuration/tests/testdata.py b/contentcuration/contentcuration/tests/testdata.py index e938b3b237..602f9d948d 100644 --- a/contentcuration/contentcuration/tests/testdata.py +++ b/contentcuration/contentcuration/tests/testdata.py @@ -206,6 +206,25 @@ def node(data, parent=None): # noqa: C901 return new_node +def country(name="Test Country", code="TC"): + return mixer.blend(cc.Country, name=name, code=code) + + +def community_library_submission(): + channel_obj = channel(name=random_string()) + user_obj = user(email=random_string()) + channel_obj.editors.add(user_obj) + channel_obj.save() + + return mixer.blend( + cc.CommunityLibrarySubmission, + channel=channel_obj, + author=user_obj, + status=cc.CommunityLibrarySubmission.Status.PENDING, + categories=list(), + ) + + def tree(parent=None): # Read from json fixture filepath = os.path.sep.join([os.path.dirname(__file__), "fixtures", "tree.json"]) From fcc7dbdbd20812cefcc5047ca0be564dcbc8e833 Mon Sep 17 00:00:00 2001 From: Jakub Komarek Date: Thu, 3 Jul 2025 19:12:06 +0200 Subject: [PATCH 003/472] Update CommunityLibrarySubmission to address PR feedback --- .../constants/community_library_submission.py | 11 ++++++++ .../0154_community_library_submission.py | 28 +++++++++++-------- contentcuration/contentcuration/models.py | 24 ++++++++-------- .../contentcuration/tests/test_models.py | 3 +- .../contentcuration/tests/testdata.py | 3 +- 5 files changed, 45 insertions(+), 24 deletions(-) create mode 100644 contentcuration/contentcuration/constants/community_library_submission.py diff --git a/contentcuration/contentcuration/constants/community_library_submission.py b/contentcuration/contentcuration/constants/community_library_submission.py new file mode 100644 index 0000000000..e477cbbb3c --- /dev/null +++ b/contentcuration/contentcuration/constants/community_library_submission.py @@ -0,0 +1,11 @@ +STATUS_PENDING = "PENDING" +STATUS_APPROVED = "APPROVED" +STATUS_REJECTED = "REJECTED" +STATUS_LIVE = "LIVE" + +COMMUNITY_LIBRARY_SUBMISSION_STATUS_CHOICES = ( + (STATUS_PENDING, "Pending"), + (STATUS_APPROVED, "Approved"), + (STATUS_REJECTED, "Rejected"), + (STATUS_LIVE, "Live"), +) diff --git a/contentcuration/contentcuration/migrations/0154_community_library_submission.py b/contentcuration/contentcuration/migrations/0154_community_library_submission.py index e70df7e82b..1816dce788 100644 --- a/contentcuration/contentcuration/migrations/0154_community_library_submission.py +++ b/contentcuration/contentcuration/migrations/0154_community_library_submission.py @@ -1,4 +1,4 @@ -# Generated by Django 3.2.24 on 2025-07-02 20:48 +# Generated by Django 3.2.24 on 2025-07-03 17:06 import django.db.models.deletion from django.conf import settings from django.db import migrations @@ -39,21 +39,21 @@ class Migration(migrations.Migration): verbose_name="ID", ), ), - ("description", models.TextField(blank=True, max_length=400)), - ("channel_version", models.IntegerField()), - ("categories", models.JSONField(blank=True, default=list)), + ("description", models.TextField(blank=True)), + ("channel_version", models.PositiveIntegerField()), + ("categories", models.JSONField(blank=True, default=list, null=True)), ("date_created", models.DateTimeField(auto_now_add=True)), ( "status", models.CharField( choices=[ - ("P", "Pending"), - ("F", "Flagged for review"), - ("A", "Approved"), - ("L", "Live"), + ("PENDING", "Pending"), + ("APPROVED", "Approved"), + ("REJECTED", "Rejected"), + ("LIVE", "Live"), ], - default="P", - max_length=1, + default="PENDING", + max_length=20, ), ), ( @@ -72,7 +72,13 @@ class Migration(migrations.Migration): to="contentcuration.channel", ), ), - ("countries", models.ManyToManyField(to="contentcuration.Country")), + ( + "countries", + models.ManyToManyField( + related_name="community_library_submissions", + to="contentcuration.Country", + ), + ), ], ), migrations.AddConstraint( diff --git a/contentcuration/contentcuration/models.py b/contentcuration/contentcuration/models.py index ee07197869..a8c78861c5 100644 --- a/contentcuration/contentcuration/models.py +++ b/contentcuration/contentcuration/models.py @@ -69,6 +69,10 @@ from contentcuration.constants import completion_criteria from contentcuration.constants import feedback from contentcuration.constants import user_history +from contentcuration.constants.community_library_submission import ( + COMMUNITY_LIBRARY_SUBMISSION_STATUS_CHOICES, +) +from contentcuration.constants.community_library_submission import STATUS_PENDING from contentcuration.constants.contentnode import kind_activity_map from contentcuration.db.models.expressions import Array from contentcuration.db.models.functions import ArrayRemove @@ -2540,29 +2544,27 @@ class Country(models.Model): class CommunityLibrarySubmission(models.Model): - class Status(models.TextChoices): - PENDING = "P", _("Pending") - FLAGGED = "F", _("Flagged for review") - APPROVED = "A", _("Approved") - LIVE = "L", _("Live") - - description = models.TextField(max_length=400, blank=True) + description = models.TextField(blank=True) channel = models.ForeignKey( Channel, related_name="community_library_submissions", on_delete=models.CASCADE, ) - channel_version = models.IntegerField() + channel_version = models.PositiveIntegerField() author = models.ForeignKey( User, related_name="community_library_submissions", on_delete=models.CASCADE, ) - countries = models.ManyToManyField(Country) - categories = models.JSONField(default=list, blank=True) + countries = models.ManyToManyField( + Country, related_name="community_library_submissions" + ) + categories = models.JSONField(default=list, blank=True, null=True) date_created = models.DateTimeField(auto_now_add=True) status = models.CharField( - max_length=1, choices=Status.choices, default=Status.PENDING + max_length=20, + choices=COMMUNITY_LIBRARY_SUBMISSION_STATUS_CHOICES, + default=STATUS_PENDING, ) def clean(self): diff --git a/contentcuration/contentcuration/tests/test_models.py b/contentcuration/contentcuration/tests/test_models.py index aecf725c64..28a05bc7cf 100644 --- a/contentcuration/contentcuration/tests/test_models.py +++ b/contentcuration/contentcuration/tests/test_models.py @@ -13,6 +13,7 @@ from contentcuration.constants import channel_history from contentcuration.constants import user_history +from contentcuration.constants.community_library_submission import STATUS_PENDING from contentcuration.models import AssessmentItem from contentcuration.models import Change from contentcuration.models import Channel @@ -568,7 +569,7 @@ def test_create_submission(self): channel_version=1, author=author, categories=["test_category"], - status=CommunityLibrarySubmission.Status.PENDING, + status=STATUS_PENDING, ) submission.countries.add(country) diff --git a/contentcuration/contentcuration/tests/testdata.py b/contentcuration/contentcuration/tests/testdata.py index 602f9d948d..320e8fc91b 100644 --- a/contentcuration/contentcuration/tests/testdata.py +++ b/contentcuration/contentcuration/tests/testdata.py @@ -16,6 +16,7 @@ from PIL import Image from contentcuration import models as cc +from contentcuration.constants.community_library_submission import STATUS_PENDING from contentcuration.tests.utils import mixer pytestmark = pytest.mark.django_db @@ -220,7 +221,7 @@ def community_library_submission(): cc.CommunityLibrarySubmission, channel=channel_obj, author=user_obj, - status=cc.CommunityLibrarySubmission.Status.PENDING, + status=STATUS_PENDING, categories=list(), ) From c296df1f512bb808ff643f143dfd54c35ea6a637 Mon Sep 17 00:00:00 2001 From: Jakub Komarek Date: Fri, 4 Jul 2025 10:58:00 +0200 Subject: [PATCH 004/472] Don't use list as default for categories --- .../migrations/0154_community_library_submission.py | 2 +- contentcuration/contentcuration/models.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/contentcuration/contentcuration/migrations/0154_community_library_submission.py b/contentcuration/contentcuration/migrations/0154_community_library_submission.py index 1816dce788..051a81a48d 100644 --- a/contentcuration/contentcuration/migrations/0154_community_library_submission.py +++ b/contentcuration/contentcuration/migrations/0154_community_library_submission.py @@ -41,7 +41,7 @@ class Migration(migrations.Migration): ), ("description", models.TextField(blank=True)), ("channel_version", models.PositiveIntegerField()), - ("categories", models.JSONField(blank=True, default=list, null=True)), + ("categories", models.JSONField(blank=True, null=True)), ("date_created", models.DateTimeField(auto_now_add=True)), ( "status", diff --git a/contentcuration/contentcuration/models.py b/contentcuration/contentcuration/models.py index a8c78861c5..003acc22f7 100644 --- a/contentcuration/contentcuration/models.py +++ b/contentcuration/contentcuration/models.py @@ -2559,7 +2559,7 @@ class CommunityLibrarySubmission(models.Model): countries = models.ManyToManyField( Country, related_name="community_library_submissions" ) - categories = models.JSONField(default=list, blank=True, null=True) + categories = models.JSONField(blank=True, null=True) date_created = models.DateTimeField(auto_now_add=True) status = models.CharField( max_length=20, From dc508354719c42bf544bff5b5b410f6b221b0731 Mon Sep 17 00:00:00 2001 From: Jakub Komarek Date: Fri, 4 Jul 2025 11:26:18 +0200 Subject: [PATCH 005/472] Import community library submission constants as a whole module --- .../constants/community_library_submission.py | 2 +- contentcuration/contentcuration/models.py | 9 +++------ contentcuration/contentcuration/tests/test_models.py | 4 ++-- contentcuration/contentcuration/tests/testdata.py | 7 +++++-- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/contentcuration/contentcuration/constants/community_library_submission.py b/contentcuration/contentcuration/constants/community_library_submission.py index e477cbbb3c..640b804f92 100644 --- a/contentcuration/contentcuration/constants/community_library_submission.py +++ b/contentcuration/contentcuration/constants/community_library_submission.py @@ -3,7 +3,7 @@ STATUS_REJECTED = "REJECTED" STATUS_LIVE = "LIVE" -COMMUNITY_LIBRARY_SUBMISSION_STATUS_CHOICES = ( +status_choices = ( (STATUS_PENDING, "Pending"), (STATUS_APPROVED, "Approved"), (STATUS_REJECTED, "Rejected"), diff --git a/contentcuration/contentcuration/models.py b/contentcuration/contentcuration/models.py index 003acc22f7..f0393b6cf1 100644 --- a/contentcuration/contentcuration/models.py +++ b/contentcuration/contentcuration/models.py @@ -66,13 +66,10 @@ from rest_framework.utils.encoders import JSONEncoder from contentcuration.constants import channel_history +from contentcuration.constants import community_library_submission from contentcuration.constants import completion_criteria from contentcuration.constants import feedback from contentcuration.constants import user_history -from contentcuration.constants.community_library_submission import ( - COMMUNITY_LIBRARY_SUBMISSION_STATUS_CHOICES, -) -from contentcuration.constants.community_library_submission import STATUS_PENDING from contentcuration.constants.contentnode import kind_activity_map from contentcuration.db.models.expressions import Array from contentcuration.db.models.functions import ArrayRemove @@ -2563,8 +2560,8 @@ class CommunityLibrarySubmission(models.Model): date_created = models.DateTimeField(auto_now_add=True) status = models.CharField( max_length=20, - choices=COMMUNITY_LIBRARY_SUBMISSION_STATUS_CHOICES, - default=STATUS_PENDING, + choices=community_library_submission.status_choices, + default=community_library_submission.STATUS_PENDING, ) def clean(self): diff --git a/contentcuration/contentcuration/tests/test_models.py b/contentcuration/contentcuration/tests/test_models.py index 28a05bc7cf..b2e5377f60 100644 --- a/contentcuration/contentcuration/tests/test_models.py +++ b/contentcuration/contentcuration/tests/test_models.py @@ -12,8 +12,8 @@ from le_utils.constants import format_presets from contentcuration.constants import channel_history +from contentcuration.constants import community_library_submission from contentcuration.constants import user_history -from contentcuration.constants.community_library_submission import STATUS_PENDING from contentcuration.models import AssessmentItem from contentcuration.models import Change from contentcuration.models import Channel @@ -569,7 +569,7 @@ def test_create_submission(self): channel_version=1, author=author, categories=["test_category"], - status=STATUS_PENDING, + status=community_library_submission.STATUS_PENDING, ) submission.countries.add(country) diff --git a/contentcuration/contentcuration/tests/testdata.py b/contentcuration/contentcuration/tests/testdata.py index 320e8fc91b..55efcd893f 100644 --- a/contentcuration/contentcuration/tests/testdata.py +++ b/contentcuration/contentcuration/tests/testdata.py @@ -16,9 +16,12 @@ from PIL import Image from contentcuration import models as cc -from contentcuration.constants.community_library_submission import STATUS_PENDING +from contentcuration.constants import ( + community_library_submission as community_library_submission_constants, +) from contentcuration.tests.utils import mixer + pytestmark = pytest.mark.django_db @@ -221,7 +224,7 @@ def community_library_submission(): cc.CommunityLibrarySubmission, channel=channel_obj, author=user_obj, - status=STATUS_PENDING, + status=community_library_submission_constants.STATUS_PENDING, categories=list(), ) From 7f1751219da43c2ce363bc8cf082ca9b27beeb0e Mon Sep 17 00:00:00 2001 From: Jakub Komarek Date: Mon, 7 Jul 2025 22:02:38 +0200 Subject: [PATCH 006/472] Move author validation to save method --- contentcuration/contentcuration/models.py | 16 +++++++--------- .../contentcuration/tests/test_models.py | 14 +------------- 2 files changed, 8 insertions(+), 22 deletions(-) diff --git a/contentcuration/contentcuration/models.py b/contentcuration/contentcuration/models.py index f0393b6cf1..70cf3f6b69 100644 --- a/contentcuration/contentcuration/models.py +++ b/contentcuration/contentcuration/models.py @@ -2564,23 +2564,21 @@ class CommunityLibrarySubmission(models.Model): default=community_library_submission.STATUS_PENDING, ) - def clean(self): - """ - Validate that the submission author is an editor of the channel. - This cannot be expressed as a constraint because traversing - related fields is not supported in constraints. Note that this - method is NOT called automatically on every save. - """ - super().clean() + def save(self, *args, **kwargs): + # Validate on save that the submission author is an editor of the channel. + # This cannot be expressed as a constraint because traversing + # related fields is not supported in constraints. if not self.channel.editors.filter(pk=self.author.pk).exists(): raise ValidationError( _( "The submission author must be an editor of the channel the submission " - "belong to" + "belongs to" ), code="author_not_editor", ) + super().save(*args, **kwargs) + @classmethod def filter_view_queryset(cls, queryset, user): if user.is_anonymous: diff --git a/contentcuration/contentcuration/tests/test_models.py b/contentcuration/contentcuration/tests/test_models.py index b2e5377f60..5cd26454eb 100644 --- a/contentcuration/contentcuration/tests/test_models.py +++ b/contentcuration/contentcuration/tests/test_models.py @@ -642,7 +642,7 @@ def test_filter_edit_queryset__channel_editor(self): ) self.assertFalse(queryset.exists()) - def test_filter_edit_queryset__author__channel_editor(self): + def test_filter_edit_queryset__author(self): submission_a = testdata.community_library_submission() submission_b = testdata.community_library_submission() @@ -652,18 +652,6 @@ def test_filter_edit_queryset__author__channel_editor(self): self.assertQuerysetContains(queryset, pk=submission_a.id) self.assertQuerysetDoesNotContain(queryset, pk=submission_b.id) - def test_filter_edit_queryset__author__not_channel_editor(self): - submission = testdata.community_library_submission() - - user = testdata.user(email="new@user.com") - submission.author = user - submission.save() - - queryset = CommunityLibrarySubmission.filter_edit_queryset( - self.base_queryset, user=user - ) - self.assertFalse(queryset.exists()) - def test_filter_edit_queryset__admin(self): submission_a = testdata.community_library_submission() From f0a05ebe7690811b2fb223861e068a82046dc008 Mon Sep 17 00:00:00 2001 From: Jakub Komarek Date: Mon, 7 Jul 2025 22:42:59 +0200 Subject: [PATCH 007/472] Add channel version checks --- contentcuration/contentcuration/models.py | 18 ++++++++++-- .../contentcuration/tests/test_models.py | 29 +++++++++++++++++++ .../contentcuration/tests/testdata.py | 2 ++ 3 files changed, 47 insertions(+), 2 deletions(-) diff --git a/contentcuration/contentcuration/models.py b/contentcuration/contentcuration/models.py index 70cf3f6b69..d395711261 100644 --- a/contentcuration/contentcuration/models.py +++ b/contentcuration/contentcuration/models.py @@ -2565,8 +2565,9 @@ class CommunityLibrarySubmission(models.Model): ) def save(self, *args, **kwargs): - # Validate on save that the submission author is an editor of the channel. - # This cannot be expressed as a constraint because traversing + # Validate on save that the submission author is an editor of the channel + # and that the version is not greater than the current channel version. + # These cannot be expressed as constraints because traversing # related fields is not supported in constraints. if not self.channel.editors.filter(pk=self.author.pk).exists(): raise ValidationError( @@ -2577,6 +2578,19 @@ def save(self, *args, **kwargs): code="author_not_editor", ) + if self.channel_version <= 0: + raise ValidationError( + _("Channel version must be positive"), + code="non_positive_channel_version", + ) + if self.channel_version > self.channel.version: + raise ValidationError( + _( + "Channel version must be less than or equal to the current channel version" + ), + code="impossibly_high_channel_version", + ) + super().save(*args, **kwargs) @classmethod diff --git a/contentcuration/contentcuration/tests/test_models.py b/contentcuration/contentcuration/tests/test_models.py index 5cd26454eb..d70bdc5d8d 100644 --- a/contentcuration/contentcuration/tests/test_models.py +++ b/contentcuration/contentcuration/tests/test_models.py @@ -559,6 +559,7 @@ def test_create_submission(self): channel = testdata.channel() author = testdata.user() channel.editors.add(author) + channel.version = 1 channel.save() country = testdata.country() @@ -576,6 +577,34 @@ def test_create_submission(self): submission.full_clean() submission.save() + def test_save__author_not_editor(self): + submission = testdata.community_library_submission() + user = testdata.user("some@email.com") + submission.author = user + with self.assertRaises(ValidationError): + submission.save() + + def test_save__nonpositive_channel_version(self): + submission = testdata.community_library_submission() + submission.channel_version = 0 + with self.assertRaises(ValidationError): + submission.save() + + def test_save__matching_channel_version(self): + submission = testdata.community_library_submission() + submission.channel.version = 5 + submission.channel.save() + submission.channel_version = 5 + submission.save() + + def test_save__impossibly_high_channel_version(self): + submission = testdata.community_library_submission() + submission.channel.version = 5 + submission.channel.save() + submission.channel_version = 6 + with self.assertRaises(ValidationError): + submission.save() + def test_filter_view_queryset__anonymous(self): _ = testdata.community_library_submission() diff --git a/contentcuration/contentcuration/tests/testdata.py b/contentcuration/contentcuration/tests/testdata.py index 55efcd893f..22e52eb0d4 100644 --- a/contentcuration/contentcuration/tests/testdata.py +++ b/contentcuration/contentcuration/tests/testdata.py @@ -218,6 +218,7 @@ def community_library_submission(): channel_obj = channel(name=random_string()) user_obj = user(email=random_string()) channel_obj.editors.add(user_obj) + channel_obj.version = 1 channel_obj.save() return mixer.blend( @@ -226,6 +227,7 @@ def community_library_submission(): author=user_obj, status=community_library_submission_constants.STATUS_PENDING, categories=list(), + channel_version=1, ) From cb61cb11f6e0794882de0d64b7a84ff9390f7528 Mon Sep 17 00:00:00 2001 From: Jakub Komarek Date: Tue, 8 Jul 2025 17:53:44 +0200 Subject: [PATCH 008/472] Do not use translation for validation errors --- contentcuration/contentcuration/models.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/contentcuration/contentcuration/models.py b/contentcuration/contentcuration/models.py index d395711261..375cd3a9a6 100644 --- a/contentcuration/contentcuration/models.py +++ b/contentcuration/contentcuration/models.py @@ -2571,23 +2571,19 @@ def save(self, *args, **kwargs): # related fields is not supported in constraints. if not self.channel.editors.filter(pk=self.author.pk).exists(): raise ValidationError( - _( - "The submission author must be an editor of the channel the submission " - "belongs to" - ), + "The submission author must be an editor of the channel the submission " + "belongs to", code="author_not_editor", ) if self.channel_version <= 0: raise ValidationError( - _("Channel version must be positive"), + "Channel version must be positive", code="non_positive_channel_version", ) if self.channel_version > self.channel.version: raise ValidationError( - _( - "Channel version must be less than or equal to the current channel version" - ), + "Channel version must be less than or equal to the current channel version", code="impossibly_high_channel_version", ) From 9e7c0f7fff6f749bc04b8acb8635f848eaa5823d Mon Sep 17 00:00:00 2001 From: Jakub Komarek Date: Wed, 9 Jul 2025 23:13:11 +0200 Subject: [PATCH 009/472] CommunityLibrarySubmission viewset --- ...ysubmission_submission_date_created_idx.py | 19 + contentcuration/contentcuration/models.py | 5 + .../test_community_library_submission.py | 408 ++++++++++++++++++ contentcuration/contentcuration/urls.py | 8 + .../viewsets/community_library_submission.py | 145 +++++++ 5 files changed, 585 insertions(+) create mode 100644 contentcuration/contentcuration/migrations/0155_communitylibrarysubmission_submission_date_created_idx.py create mode 100644 contentcuration/contentcuration/tests/viewsets/test_community_library_submission.py create mode 100644 contentcuration/contentcuration/viewsets/community_library_submission.py diff --git a/contentcuration/contentcuration/migrations/0155_communitylibrarysubmission_submission_date_created_idx.py b/contentcuration/contentcuration/migrations/0155_communitylibrarysubmission_submission_date_created_idx.py new file mode 100644 index 0000000000..65e7a8bb40 --- /dev/null +++ b/contentcuration/contentcuration/migrations/0155_communitylibrarysubmission_submission_date_created_idx.py @@ -0,0 +1,19 @@ +# Generated by Django 3.2.24 on 2025-07-08 20:09 +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + + dependencies = [ + ("contentcuration", "0154_community_library_submission"), + ] + + operations = [ + migrations.AddIndex( + model_name="communitylibrarysubmission", + index=models.Index( + fields=["-date_created"], name="submission_date_created_idx" + ), + ), + ] diff --git a/contentcuration/contentcuration/models.py b/contentcuration/contentcuration/models.py index 375cd3a9a6..6d0e0e93a8 100644 --- a/contentcuration/contentcuration/models.py +++ b/contentcuration/contentcuration/models.py @@ -2616,6 +2616,10 @@ class Meta: name="unique_channel_with_channel_version", ), ] + indexes = [ + # Useful for cursor pagination + models.Index(fields=["-date_created"], name="submission_date_created_idx"), + ] ASSESSMENT_ID_INDEX_NAME = "assessment_id_idx" @@ -2655,6 +2659,7 @@ def has_changes(self): class Meta: indexes = [ + # Useful for cursor pagination models.Index(fields=["assessment_id"], name=ASSESSMENT_ID_INDEX_NAME), ] diff --git a/contentcuration/contentcuration/tests/viewsets/test_community_library_submission.py b/contentcuration/contentcuration/tests/viewsets/test_community_library_submission.py new file mode 100644 index 0000000000..8cb9e5e7c2 --- /dev/null +++ b/contentcuration/contentcuration/tests/viewsets/test_community_library_submission.py @@ -0,0 +1,408 @@ +from urllib.parse import urlencode + +from django.urls import reverse + +from contentcuration.models import CommunityLibrarySubmission +from contentcuration.tests import testdata +from contentcuration.tests.base import StudioAPITestCase + + +def reverse_with_query( + viewname, urlconf=None, args=None, kwargs=None, current_app=None, query=None +): + """ + This helper wraps the Django `reverse` function to support the `query` argument. + This argument is supported natively since Django 5.2, so when Django is updated + above this version, this helper can be removed. + """ + url = reverse( + viewname, urlconf=urlconf, args=args, kwargs=kwargs, current_app=current_app + ) + if query: + return f"{url}?{urlencode(query)}" + return url + + +class CRUDTestCase(StudioAPITestCase): + @property + def new_submission_metadata(self): + return { + "channel": self.channel_without_submission.id, + "countries": [self.country1.code], + } + + @property + def updated_submission_metadata(self): + return { + "countries": [ + "C2", + ], + "channel": self.channel_with_submission1.id, + } + + def setUp(self): + super().setUp() + self.author_user = testdata.user(email="author@user.com") + self.editor_user = testdata.user(email="editor@user.com") + self.forbidden_user = testdata.user(email="forbidden@user.com") + self.admin_user = testdata.user(email="admin@user.com") + self.admin_user.is_admin = True + self.admin_user.save() + + self.country1 = testdata.country(name="Country 1", code="C1") + self.country2 = testdata.country(name="Country 2", code="C2") + + self.channel_with_submission1 = testdata.channel() + self.channel_with_submission1.public = True + self.channel_with_submission1.version = 1 + self.channel_with_submission1.editors.add(self.author_user) + self.channel_with_submission1.editors.add(self.editor_user) + self.channel_with_submission1.save() + + self.channel_with_submission2 = testdata.channel() + self.channel_with_submission2.public = True + self.channel_with_submission2.version = 1 + self.channel_with_submission2.editors.add(self.author_user) + self.channel_with_submission2.editors.add(self.editor_user) + self.channel_with_submission2.save() + + self.channel_without_submission = testdata.channel() + self.channel_without_submission.public = True + self.channel_without_submission.version = 1 + self.channel_without_submission.editors.add(self.author_user) + self.channel_without_submission.editors.add(self.editor_user) + self.channel_without_submission.save() + + self.unpublished_channel = testdata.channel() + self.unpublished_channel.public = False + self.unpublished_channel.version = 0 + self.unpublished_channel.editors.add(self.author_user) + self.unpublished_channel.editors.add(self.editor_user) + self.unpublished_channel.save() + + self.existing_submission1 = testdata.community_library_submission() + self.existing_submission1.channel = self.channel_with_submission1 + self.existing_submission1.author = self.author_user + self.existing_submission1.save() + self.existing_submission1.countries.add(self.country1) + self.existing_submission1.save() + + self.existing_submission2 = testdata.community_library_submission() + self.existing_submission2.channel = self.channel_with_submission2 + self.existing_submission2.author = self.author_user + self.existing_submission2.save() + self.existing_submission2.countries.add(self.country1) + self.existing_submission2.save() + + def test_create_submission__is_editor(self): + self.client.force_authenticate(user=self.editor_user) + submission = self.new_submission_metadata + response = self.client.post( + reverse("community-library-submission-list"), + submission, + format="json", + ) + self.assertEqual(response.status_code, 201, response.content) + + def test_create_submission__is_forbidden(self): + self.client.force_authenticate(user=self.forbidden_user) + submission = self.new_submission_metadata + response = self.client.post( + reverse("community-library-submission-list"), + submission, + format="json", + ) + self.assertEqual(response.status_code, 400, response.content) + + def test_create_submission__unpublished_channel(self): + self.client.force_authenticate(user=self.editor_user) + submission = self.new_submission_metadata + submission["channel"] = self.unpublished_channel.id + + response = self.client.post( + reverse("community-library-submission-list"), + submission, + format="json", + ) + self.assertEqual(response.status_code, 400, response.content) + + def test_create_submission__explicit_channel_version(self): + self.client.force_authenticate(user=self.editor_user) + submission_metadata = self.new_submission_metadata + submission_metadata["channel_version"] = 2 + response = self.client.post( + reverse("community-library-submission-list"), + submission_metadata, + format="json", + ) + self.assertEqual(response.status_code, 201, response.content) + + created_submission_id = response.data["id"] + created_submission = CommunityLibrarySubmission.objects.get( + id=created_submission_id + ) + + # The explicitly set channel version should be ignored by the serializer + self.assertEqual(created_submission.channel_version, 1) + + def test_list_submissions__is_editor(self): + self.client.force_authenticate(user=self.editor_user) + response = self.client.get( + reverse( + "community-library-submission-list", + ) + ) + self.assertEqual(response.status_code, 200, response.content) + + results = response.data + self.assertEqual(len(results), 2) + + def test_list_submissions__is_admin(self): + self.client.force_authenticate(user=self.admin_user) + response = self.client.get( + reverse( + "community-library-submission-list", + ) + ) + self.assertEqual(response.status_code, 200, response.content) + + results = response.data + self.assertEqual(len(results), 2) + + def test_list_submissions__is_forbidden(self): + self.client.force_authenticate(user=self.forbidden_user) + response = self.client.get( + reverse( + "community-library-submission-list", + ) + ) + self.assertEqual(response.status_code, 200, response.content) + + results = response.data + self.assertEqual(len(results), 0) + + def test_list_submissions__filter_by_channel(self): + self.client.force_authenticate(user=self.editor_user) + response = self.client.get( + reverse_with_query( + "community-library-submission-list", + query={"channel": self.channel_with_submission1.id}, + ) + ) + self.assertEqual(response.status_code, 200, response.content) + + results = response.data + self.assertEqual(len(results), 1) + self.assertEqual(results[0]["channel_id"], self.channel_with_submission1.id) + + def test_list_submissions__pagination(self): + self.client.force_authenticate(user=self.author_user) + response = self.client.get( + reverse_with_query( + "community-library-submission-list", + query={"max_results": 1}, + ) + ) + self.assertEqual(response.status_code, 200, response.content) + + results = response.data["results"] + more = response.data["more"] + + self.assertEqual(len(results), 1) + self.assertIsNotNone(more) + + cursor = more["cursor"] + + response = self.client.get( + reverse_with_query( + "community-library-submission-list", + query={"max_results": 1, "cursor": cursor}, + ) + ) + self.assertEqual(response.status_code, 200, response.content) + + results = response.data["results"] + more = response.data["more"] + + self.assertEqual(len(results), 1) + self.assertIsNone(more) + + def test_get_single_submission__is_editor(self): + self.client.force_authenticate(user=self.editor_user) + response = self.client.get( + reverse( + "community-library-submission-detail", + args=[self.existing_submission1.id], + ), + format="json", + ) + self.assertEqual(response.status_code, 200, response.content) + + result = response.data + self.assertEqual(result["id"], self.existing_submission1.id) + + def test_get_single_submission__is_admin(self): + self.client.force_authenticate(user=self.admin_user) + response = self.client.get( + reverse( + "community-library-submission-detail", + args=[self.existing_submission1.id], + ), + format="json", + ) + self.assertEqual(response.status_code, 200, response.content) + + result = response.data + self.assertEqual(result["id"], self.existing_submission1.id) + + def test_get_single_submission__is_forbidden(self): + self.client.force_authenticate(user=self.forbidden_user) + response = self.client.get( + reverse( + "community-library-submission-detail", + args=[self.existing_submission1.id], + ), + format="json", + ) + self.assertEqual(response.status_code, 404, response.content) + + def test_get_single_submission__author_name(self): + self.client.force_authenticate(user=self.author_user) + response = self.client.get( + reverse( + "community-library-submission-detail", + args=[self.existing_submission1.id], + ), + format="json", + ) + self.assertEqual(response.status_code, 200, response.content) + + result = response.data + self.assertEqual(result["author_first_name"], self.author_user.first_name) + self.assertEqual(result["author_last_name"], self.author_user.last_name) + + def test_update_submission__is_author(self): + self.client.force_authenticate(user=self.author_user) + response = self.client.put( + reverse( + "community-library-submission-detail", + args=[self.existing_submission1.id], + ), + self.updated_submission_metadata, + format="json", + ) + self.assertEqual(response.status_code, 200, response.content) + + updated_submission = CommunityLibrarySubmission.objects.get( + id=self.existing_submission1.id + ) + self.assertEqual(updated_submission.countries.count(), 1) + self.assertEqual(updated_submission.countries.first().code, "C2") + + def test_update_submission__is_editor(self): + self.client.force_authenticate(user=self.editor_user) + response = self.client.put( + reverse( + "community-library-submission-detail", + args=[self.existing_submission1.id], + ), + self.updated_submission_metadata, + format="json", + ) + self.assertEqual(response.status_code, 404, response.content) + + def test_update_submission__is_admin(self): + self.client.force_authenticate(user=self.admin_user) + response = self.client.put( + reverse( + "community-library-submission-detail", + args=[self.existing_submission1.id], + ), + self.updated_submission_metadata, + format="json", + ) + self.assertEqual(response.status_code, 200, response.content) + + updated_submission = CommunityLibrarySubmission.objects.get( + id=self.existing_submission1.id + ) + self.assertEqual(updated_submission.countries.count(), 1) + self.assertEqual(updated_submission.countries.first().code, "C2") + + def test_update_submission__change_channel(self): + self.client.force_authenticate(user=self.admin_user) + submission_metadata = self.updated_submission_metadata + submission_metadata["channel"] = self.channel_without_submission.id + response = self.client.put( + reverse( + "community-library-submission-detail", + args=[self.existing_submission1.id], + ), + submission_metadata, + format="json", + ) + self.assertEqual(response.status_code, 400, response.content) + + def test_delete_submission__is_author(self): + self.client.force_authenticate(user=self.author_user) + response = self.client.delete( + reverse( + "community-library-submission-detail", + args=[self.existing_submission1.id], + ), + format="json", + ) + self.assertEqual(response.status_code, 204, response.content) + self.assertFalse( + CommunityLibrarySubmission.objects.filter( + id=self.existing_submission1.id + ).exists() + ) + + def test_delete_submission__is_editor(self): + self.client.force_authenticate(user=self.editor_user) + response = self.client.delete( + reverse( + "community-library-submission-detail", + args=[self.existing_submission1.id], + ), + format="json", + ) + self.assertEqual(response.status_code, 404, response.content) + self.assertTrue( + CommunityLibrarySubmission.objects.filter( + id=self.existing_submission1.id + ).exists() + ) + + def test_delete_submission__is_admin(self): + self.client.force_authenticate(user=self.admin_user) + response = self.client.delete( + reverse( + "community-library-submission-detail", + args=[self.existing_submission1.id], + ), + format="json", + ) + self.assertEqual(response.status_code, 204, response.content) + self.assertFalse( + CommunityLibrarySubmission.objects.filter( + id=self.existing_submission1.id + ).exists() + ) + + def test_delete_submission__is_forbidden(self): + self.client.force_authenticate(user=self.forbidden_user) + response = self.client.delete( + reverse( + "community-library-submission-detail", + args=[self.existing_submission1.id], + ), + format="json", + ) + self.assertEqual(response.status_code, 404, response.content) + self.assertTrue( + CommunityLibrarySubmission.objects.filter( + id=self.existing_submission1.id + ).exists() + ) diff --git a/contentcuration/contentcuration/urls.py b/contentcuration/contentcuration/urls.py index 94fe8511e3..b948effc8b 100644 --- a/contentcuration/contentcuration/urls.py +++ b/contentcuration/contentcuration/urls.py @@ -38,6 +38,9 @@ from contentcuration.viewsets.channel import ChannelViewSet from contentcuration.viewsets.channelset import ChannelSetViewSet from contentcuration.viewsets.clipboard import ClipboardViewSet +from contentcuration.viewsets.community_library_submission import ( + CommunityLibrarySubmissionViewSet, +) from contentcuration.viewsets.contentnode import ContentNodeViewSet from contentcuration.viewsets.feedback import FlagFeedbackEventViewSet from contentcuration.viewsets.feedback import RecommendationsEventViewSet @@ -80,6 +83,11 @@ def get_redirect_url(self, *args, **kwargs): RecommendationsInteractionEventViewSet, basename="recommendations-interaction", ) +router.register( + r"communitylibrary_submission", + CommunityLibrarySubmissionViewSet, + basename="community-library-submission", +) urlpatterns = [ re_path(r"^api/", include(router.urls)), diff --git a/contentcuration/contentcuration/viewsets/community_library_submission.py b/contentcuration/contentcuration/viewsets/community_library_submission.py new file mode 100644 index 0000000000..907460752c --- /dev/null +++ b/contentcuration/contentcuration/viewsets/community_library_submission.py @@ -0,0 +1,145 @@ +from django_filters.rest_framework import DjangoFilterBackend +from rest_framework.permissions import IsAuthenticated +from rest_framework.relations import PrimaryKeyRelatedField +from rest_framework.serializers import ValidationError + +from contentcuration.models import Channel +from contentcuration.models import CommunityLibrarySubmission +from contentcuration.models import Country +from contentcuration.utils.pagination import ValuesViewsetCursorPagination +from contentcuration.viewsets.base import BulkListSerializer +from contentcuration.viewsets.base import BulkModelSerializer +from contentcuration.viewsets.base import ReadOnlyValuesViewset +from contentcuration.viewsets.base import RESTCreateModelMixin +from contentcuration.viewsets.base import RESTDestroyModelMixin +from contentcuration.viewsets.base import RESTUpdateModelMixin +from contentcuration.viewsets.common import UserFilteredPrimaryKeyRelatedField + + +class CommunityLibrarySubmissionSerializer(BulkModelSerializer): + countries = PrimaryKeyRelatedField( + many=True, + queryset=Country.objects.all(), + required=False, + ) + channel = UserFilteredPrimaryKeyRelatedField( + queryset=Channel.objects.all(), + edit=False, + ) + + class Meta: + model = CommunityLibrarySubmission + fields = [ + "id", + "description", + "channel", + "countries", + "categories", + ] + list_serializer_class = BulkListSerializer + + def create(self, validated_data): + channel = validated_data["channel"] + user = self.context["request"].user + + if channel.version == 0: + # The channel is not published + raise ValidationError( + "Cannot create a community library submission for an " + "unpublished channel." + ) + + if not channel.editors.filter(id=user.id).exists(): + raise ValidationError( + "Only editors can create a community library " + "submission for this channel." + ) + + validated_data["channel_version"] = channel.version + validated_data["author"] = self.context["request"].user + + countries = validated_data.pop("countries", []) + instance = super().create(validated_data) + + for country in countries: + instance.countries.add(country) + + instance.save() + return instance + + def update(self, instance, validated_data): + if instance.channel.id != validated_data["channel"].id: + raise ValidationError( + "Cannot change the channel corresponding to " + "a community library submission." + ) + + countries = validated_data.pop("countries", []) + instance.countries.set(countries) + + return super().update(instance, validated_data) + + +class CommunityLibrarySubmissionPagination(ValuesViewsetCursorPagination): + ordering = "-date_created" + page_size_query_param = "max_results" + max_page_size = 100 + + +class CommunityLibrarySubmissionViewSet( + RESTCreateModelMixin, + RESTUpdateModelMixin, + RESTDestroyModelMixin, + ReadOnlyValuesViewset, +): + values = ( + "id", + "description", + "channel_id", + "channel_version", + "author_id", + "author__first_name", + "author__last_name", + "categories", + "date_created", + "status", + ) + field_map = { + "author_first_name": "author__first_name", + "author_last_name": "author__last_name", + } + filter_backends = [DjangoFilterBackend] + filterset_fields = ["channel"] + permission_classes = [IsAuthenticated] + pagination_class = CommunityLibrarySubmissionPagination + serializer_class = CommunityLibrarySubmissionSerializer + + def get_queryset(self): + base_queryset = CommunityLibrarySubmission.objects.all() + user = self.request.user + queryset = CommunityLibrarySubmission.filter_view_queryset( + base_queryset, user + ).order_by("-date_created") + + return queryset + + def get_edit_queryset(self): + base_queryset = CommunityLibrarySubmission.objects.all() + user = self.request.user + queryset = CommunityLibrarySubmission.filter_edit_queryset(base_queryset, user) + + return queryset + + def consolidate(self, items, queryset): + countries = {} + for (submission_id, country_code,) in Country.objects.filter( + community_library_submissions__in=queryset + ).values_list("community_library_submissions", "code"): + if submission_id not in countries: + countries[submission_id] = [] + countries[submission_id].append(country_code) + + for item in items: + item["countries"] = countries.get(item["id"], []) + + return items From 0264ead88d2628050a6a486ae74d21a96562287a Mon Sep 17 00:00:00 2001 From: Jakub Komarek Date: Thu, 10 Jul 2025 11:24:45 +0200 Subject: [PATCH 010/472] Populate Country table --- .../shared/views/form/CountryField.vue | 3 ++ .../management/commands/loadconstants.py | 28 +++++++++++++++++-- requirements.in | 1 + requirements.txt | 2 ++ 4 files changed, 32 insertions(+), 2 deletions(-) diff --git a/contentcuration/contentcuration/frontend/shared/views/form/CountryField.vue b/contentcuration/contentcuration/frontend/shared/views/form/CountryField.vue index 84e3dcd53a..8254a4f8bb 100644 --- a/contentcuration/contentcuration/frontend/shared/views/form/CountryField.vue +++ b/contentcuration/contentcuration/frontend/shared/views/form/CountryField.vue @@ -30,6 +30,9 @@ import DropdownWrapper from 'shared/views/form/DropdownWrapper'; + // NOTE that this list MUST stay in sync with the list of countries + // generated on the backend in contentcuration/management/commands/loadconstants.py, + // and special care should be taken when updating the i18n-iso-countries library. var countries = require('i18n-iso-countries'); countries.registerLocale(require('i18n-iso-countries/langs/en.json')); countries.registerLocale(require('i18n-iso-countries/langs/es.json')); diff --git a/contentcuration/contentcuration/management/commands/loadconstants.py b/contentcuration/contentcuration/management/commands/loadconstants.py index 3c54ac2025..33b3907d5b 100644 --- a/contentcuration/contentcuration/management/commands/loadconstants.py +++ b/contentcuration/contentcuration/management/commands/loadconstants.py @@ -1,5 +1,6 @@ import logging as logmodule +import pycountry from django.conf import settings from django.contrib.sites.models import Site from django.core.cache import cache @@ -15,12 +16,20 @@ logging = logmodule.getLogger(__name__) +# Kosovo is supported by the i18n-iso-countries library that is used to get the +# list of countries on the frontend in +# contentcuration/frontend/shared/views/form/CountryField.vue, so we add it here +# as well to ensure that the lists generated by both libraries are the same. +# NOTE that it MUST be ensured that the lists stay in sync, and special care +# should be taken when these libraries are updated. +pycountry.countries.add_entry(alpha_2="XK", alpha_3="XXK", name="Kosovo", numeric="926") + class ConstantGenerator(object): id_field = "id" def generate_list(self): - # Get constants from subclass' default_list (from le-utils pkg) + # Get constants from subclass' default_list (usually from le-utils pkg) return [ { "model": self.model, @@ -117,6 +126,20 @@ def get_dict(self, constant): } +class CountryGenerator(ConstantGenerator): + # This does NOT use the le-utils constants module, but instead uses + # the pycountry library to get the country list directly. + id_field = "code" + default_list = pycountry.countries + model = models.Country + + def get_dict(self, constant): + return { + "code": constant.alpha_2, + "name": constant.name, + } + + SITES = [ { "model": Site, @@ -161,8 +184,9 @@ def get_dict(self, constant): KINDS = KindGenerator().generate_list() PRESETS = PresetGenerator().generate_list() LANGUAGES = LanguageGenerator().generate_list() +COUNTRIES = CountryGenerator().generate_list() -CONSTANTS = [SITES, LICENSES, KINDS, FILE_FORMATS, PRESETS, LANGUAGES] +CONSTANTS = [SITES, LICENSES, KINDS, FILE_FORMATS, PRESETS, LANGUAGES, COUNTRIES] class EarlyExit(BaseException): diff --git a/requirements.in b/requirements.in index cffdacedad..daf71b4140 100644 --- a/requirements.in +++ b/requirements.in @@ -33,3 +33,4 @@ python-dateutil>=2.8.1 jsonschema>=3.2.0 django-celery-results packaging>=21.0 +pycountry diff --git a/requirements.txt b/requirements.txt index 3ff533e5ba..761f1f8da5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -188,6 +188,8 @@ pyasn1==0.4.8 # rsa pyasn1-modules==0.2.8 # via google-auth +pycountry==24.6.1 + # via -r requirements.in pyparsing==2.4.7 # via httplib2 python-dateutil==2.9.0.post0 From 17b08380525130a33a432f3f3014df7c9ae52403 Mon Sep 17 00:00:00 2001 From: Jakub Komarek Date: Fri, 11 Jul 2025 22:03:59 +0200 Subject: [PATCH 011/472] Include licenses and categories in published channel data --- .../tests/test_exportchannel.py | 77 +++++++++++++++++-- .../contentcuration/utils/publish.py | 21 +++++ 2 files changed, 91 insertions(+), 7 deletions(-) diff --git a/contentcuration/contentcuration/tests/test_exportchannel.py b/contentcuration/contentcuration/tests/test_exportchannel.py index 57599c0942..95d75e7886 100644 --- a/contentcuration/contentcuration/tests/test_exportchannel.py +++ b/contentcuration/contentcuration/tests/test_exportchannel.py @@ -692,14 +692,77 @@ def test_nonexistent_prerequisites(self): class ChannelExportPublishedData(StudioTestCase): - def test_fill_published_fields(self): + def setUp(self): + super().setUpBase() + + self.license1 = cc.License.objects.create( + id=100, + license_name="License 1", + license_description="Description 1", + license_url="http://example.com/license1", + ) + self.license2 = cc.License.objects.create( + id=101, + license_name="License 2", + license_description="Description 2", + license_url="http://example.com/license2", + ) + + self.category1 = "Category 1" + self.category2 = "Category 2" + + self.node1 = self.channel.main_tree.get_root() + self.node2 = self.channel.main_tree.get_children().first() + self.node3 = self.node2.get_children().first() + self.node4 = self.node3.get_next_sibling() + + self.node1.license = self.license1 + self.node1.categories = {self.category1: True} + self.node1.published = True + self.node1.save() + + self.node2.license = self.license2 + self.node2.categories = {self.category2: True} + self.node2.published = True + self.node2.save() + + self.node3.license = self.license1 + self.node3.categories = {self.category1: True} + self.node3.published = True + self.node3.save() + + self.node4.license = None + self.node4.categories = None + self.node4.published = True + self.node4.save() + + def test_fill_published_fields__version_notes(self): version_notes = description() - channel = cc.Channel.objects.create(actor_id=self.admin_user.id) - channel.last_published - fill_published_fields(channel, version_notes) - self.assertTrue(channel.published_data) - self.assertIsNotNone(channel.published_data.get(0)) - self.assertEqual(channel.published_data[0]["version_notes"], version_notes) + self.channel.last_published + fill_published_fields(self.channel, version_notes) + self.assertTrue(self.channel.published_data) + self.assertIsNotNone(self.channel.published_data.get(0)) + self.assertEqual(self.channel.published_data[0]["version_notes"], version_notes) + + def test_fill_published_fields__included_licenses(self): + version_notes = description() + fill_published_fields(self.channel, version_notes) + + returned_license_ids = self.channel.published_data[0]["included_licenses"] + expected_license_ids = [self.license1.id, self.license2.id] + + self.assertEqual(len(returned_license_ids), len(expected_license_ids)) + self.assertSetEqual(set(returned_license_ids), set(expected_license_ids)) + + def test_fill_published_fields__included_categories(self): + version_notes = description() + fill_published_fields(self.channel, version_notes) + + expected_categories = [self.category1, self.category2] + returned_categories = self.channel.published_data[0]["included_categories"] + + self.assertEqual(len(returned_categories), len(expected_categories)) + self.assertSetEqual(set(returned_categories), set(expected_categories)) class PublishFailCleansUpTaskObjects(StudioTestCase): diff --git a/contentcuration/contentcuration/utils/publish.py b/contentcuration/contentcuration/utils/publish.py index 901f9aaa32..841f440134 100644 --- a/contentcuration/contentcuration/utils/publish.py +++ b/contentcuration/contentcuration/utils/publish.py @@ -1184,6 +1184,25 @@ def fill_published_fields(channel, version_notes): if lang: channel.included_languages.add(lang) + included_licenses = published_nodes.exclude(license=None).values_list( + "license", flat=True + ) + license_list = sorted(set(included_licenses)) + + included_categories_dicts = published_nodes.exclude(categories=None).values_list( + "categories", flat=True + ) + category_list = sorted( + set( + chain.from_iterable( + ( + node_categories_dict.keys() + for node_categories_dict in included_categories_dicts + ) + ) + ) + ) + # TODO: Eventually, consolidate above operations to just use this field for storing historical data channel.published_data.update( { @@ -1196,6 +1215,8 @@ def fill_published_fields(channel, version_notes): ), "version_notes": version_notes, "included_languages": language_list, + "included_licenses": license_list, + "included_categories": category_list, } } ) From 6ab6775c7ad8b8c152d0f1191d170570816d06d3 Mon Sep 17 00:00:00 2001 From: Jakub Komarek Date: Sat, 12 Jul 2025 09:37:30 +0200 Subject: [PATCH 012/472] Minor changes from PR feedback --- contentcuration/contentcuration/models.py | 1 - .../test_community_library_submission.py | 4 +--- .../viewsets/community_library_submission.py | 20 ++----------------- 3 files changed, 3 insertions(+), 22 deletions(-) diff --git a/contentcuration/contentcuration/models.py b/contentcuration/contentcuration/models.py index 6d0e0e93a8..d174261fa8 100644 --- a/contentcuration/contentcuration/models.py +++ b/contentcuration/contentcuration/models.py @@ -2659,7 +2659,6 @@ def has_changes(self): class Meta: indexes = [ - # Useful for cursor pagination models.Index(fields=["assessment_id"], name=ASSESSMENT_ID_INDEX_NAME), ] diff --git a/contentcuration/contentcuration/tests/viewsets/test_community_library_submission.py b/contentcuration/contentcuration/tests/viewsets/test_community_library_submission.py index 8cb9e5e7c2..85651dbc3e 100644 --- a/contentcuration/contentcuration/tests/viewsets/test_community_library_submission.py +++ b/contentcuration/contentcuration/tests/viewsets/test_community_library_submission.py @@ -211,12 +211,10 @@ def test_list_submissions__pagination(self): self.assertEqual(len(results), 1) self.assertIsNotNone(more) - cursor = more["cursor"] - response = self.client.get( reverse_with_query( "community-library-submission-list", - query={"max_results": 1, "cursor": cursor}, + query=more, ) ) self.assertEqual(response.status_code, 200, response.content) diff --git a/contentcuration/contentcuration/viewsets/community_library_submission.py b/contentcuration/contentcuration/viewsets/community_library_submission.py index 907460752c..b7168b1e68 100644 --- a/contentcuration/contentcuration/viewsets/community_library_submission.py +++ b/contentcuration/contentcuration/viewsets/community_library_submission.py @@ -61,8 +61,7 @@ def create(self, validated_data): countries = validated_data.pop("countries", []) instance = super().create(validated_data) - for country in countries: - instance.countries.add(country) + instance.countries.set(countries) instance.save() return instance @@ -113,22 +112,7 @@ class CommunityLibrarySubmissionViewSet( permission_classes = [IsAuthenticated] pagination_class = CommunityLibrarySubmissionPagination serializer_class = CommunityLibrarySubmissionSerializer - - def get_queryset(self): - base_queryset = CommunityLibrarySubmission.objects.all() - user = self.request.user - queryset = CommunityLibrarySubmission.filter_view_queryset( - base_queryset, user - ).order_by("-date_created") - - return queryset - - def get_edit_queryset(self): - base_queryset = CommunityLibrarySubmission.objects.all() - user = self.request.user - queryset = CommunityLibrarySubmission.filter_edit_queryset(base_queryset, user) - - return queryset + queryset = CommunityLibrarySubmission.objects.all().order_by("-date_created") def consolidate(self, items, queryset): countries = {} From ba77607f4ef2942c655d3c40797d21c8dc5b521f Mon Sep 17 00:00:00 2001 From: Jakub Komarek Date: Sat, 12 Jul 2025 10:25:38 +0200 Subject: [PATCH 013/472] Fix partial updates when channel is missing --- .../test_community_library_submission.py | 18 ++++++++++++++++++ .../viewsets/community_library_submission.py | 5 ++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/contentcuration/contentcuration/tests/viewsets/test_community_library_submission.py b/contentcuration/contentcuration/tests/viewsets/test_community_library_submission.py index 85651dbc3e..d4de9e7d6b 100644 --- a/contentcuration/contentcuration/tests/viewsets/test_community_library_submission.py +++ b/contentcuration/contentcuration/tests/viewsets/test_community_library_submission.py @@ -341,6 +341,24 @@ def test_update_submission__change_channel(self): ) self.assertEqual(response.status_code, 400, response.content) + def test_partial_update_submission__missing_channel(self): + self.client.force_authenticate(user=self.admin_user) + submission_metadata = self.updated_submission_metadata + del submission_metadata["channel"] + response = self.client.patch( + reverse( + "community-library-submission-detail", + args=[self.existing_submission1.id], + ), + submission_metadata, + format="json", + ) + self.assertEqual(response.status_code, 200, response.content) + updated_submission = CommunityLibrarySubmission.objects.get( + id=self.existing_submission1.id + ) + self.assertEqual(updated_submission.channel, self.channel_with_submission1) + def test_delete_submission__is_author(self): self.client.force_authenticate(user=self.author_user) response = self.client.delete( diff --git a/contentcuration/contentcuration/viewsets/community_library_submission.py b/contentcuration/contentcuration/viewsets/community_library_submission.py index b7168b1e68..2aef31bfba 100644 --- a/contentcuration/contentcuration/viewsets/community_library_submission.py +++ b/contentcuration/contentcuration/viewsets/community_library_submission.py @@ -67,7 +67,10 @@ def create(self, validated_data): return instance def update(self, instance, validated_data): - if instance.channel.id != validated_data["channel"].id: + if ( + "channel" in validated_data + and instance.channel.id != validated_data["channel"].id + ): raise ValidationError( "Cannot change the channel corresponding to " "a community library submission." From d839d89b272548dce0adaac75dae0f28d4cff800 Mon Sep 17 00:00:00 2001 From: Jakub Komarek Date: Sat, 12 Jul 2025 11:02:26 +0200 Subject: [PATCH 014/472] Action for getting channel published data --- .../tests/viewsets/test_channel.py | 46 +++++++++++++++++++ .../contentcuration/viewsets/channel.py | 25 ++++++++-- 2 files changed, 68 insertions(+), 3 deletions(-) diff --git a/contentcuration/contentcuration/tests/viewsets/test_channel.py b/contentcuration/contentcuration/tests/viewsets/test_channel.py index 8309f47c8c..067e442ede 100644 --- a/contentcuration/contentcuration/tests/viewsets/test_channel.py +++ b/contentcuration/contentcuration/tests/viewsets/test_channel.py @@ -940,3 +940,49 @@ def _perform_action(self, url_path, channel_id): reverse(url_path, kwargs={"pk": channel_id}), format="json" ) return response + + +class GetPublishedDataTestCase(StudioAPITestCase): + def setUp(self): + super().setUp() + + self.editor_user = testdata.user(email="editor@user.com") + self.forbidden_user = testdata.user(email="forbidden@user.com") + + self.channel = testdata.channel() + self.channel.editors.add(self.editor_user) + + self.channel.published_data = { + "key1": "value1", + "key2": "value2", + } + self.channel.save() + + def test_get_published_data__is_editor(self): + self.client.force_authenticate(user=self.editor_user) + + response = self.client.get( + reverse("channel-published-data", kwargs={"pk": self.channel.id}), + format="json", + ) + self.assertEqual(response.status_code, 200, response.content) + self.assertEqual(response.json(), self.channel.published_data) + + def test_get_published_data__is_admin(self): + self.client.force_authenticate(user=self.admin_user) + + response = self.client.get( + reverse("channel-published-data", kwargs={"pk": self.channel.id}), + format="json", + ) + self.assertEqual(response.status_code, 200, response.content) + self.assertEqual(response.json(), self.channel.published_data) + + def test_get_published_data__is_forbidden_user(self): + self.client.force_authenticate(user=self.forbidden_user) + + response = self.client.get( + reverse("channel-published-data", kwargs={"pk": self.channel.id}), + format="json", + ) + self.assertEqual(response.status_code, 404, response.content) diff --git a/contentcuration/contentcuration/viewsets/channel.py b/contentcuration/contentcuration/viewsets/channel.py index c2462836d1..f18e910794 100644 --- a/contentcuration/contentcuration/viewsets/channel.py +++ b/contentcuration/contentcuration/viewsets/channel.py @@ -774,9 +774,7 @@ def deploy(self, user, pk): url_path="language_exists", url_name="language-exists", ) - def channel_language_exists( - self, request, pk=None - ) -> Union[JsonResponse, HttpResponse]: + def channel_language_exists(self, request, pk=None) -> JsonResponse: """ Verify that the language set for a channel is present in at least one of its resources. @@ -814,6 +812,27 @@ def get_languages_in_channel( langs_in_content = self._get_channel_content_languages(pk, main_tree_id) return JsonResponse({"languages": langs_in_content}) + @action( + detail=True, + methods=["get"], + url_path="published_data", + url_name="published-data", + ) + def get_published_data(self, request, pk=None) -> Union[JsonResponse, HttpResponse]: + """ + Get the published data for a channel. + + :param request: The request object + :param pk: The ID of the channel + :return: JsonResponse with the published data of the channel + :rtype: JsonResponse + """ + # Allow exactly users with permission to edit the channel to + # access the published data. + channel = self.get_edit_object() + + return Response(channel.published_data) + def _channel_exists(self, channel_id) -> bool: """ Check if a channel exists. From 73b72be35ad74c8476fe21e18c1247b601574e68 Mon Sep 17 00:00:00 2001 From: Jakub Komarek Date: Thu, 17 Jul 2025 13:21:08 +0200 Subject: [PATCH 015/472] Fix type annotations --- contentcuration/contentcuration/viewsets/channel.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/contentcuration/contentcuration/viewsets/channel.py b/contentcuration/contentcuration/viewsets/channel.py index f18e910794..1fd6625030 100644 --- a/contentcuration/contentcuration/viewsets/channel.py +++ b/contentcuration/contentcuration/viewsets/channel.py @@ -774,7 +774,9 @@ def deploy(self, user, pk): url_path="language_exists", url_name="language-exists", ) - def channel_language_exists(self, request, pk=None) -> JsonResponse: + def channel_language_exists( + self, request, pk=None + ) -> Union[JsonResponse, HttpResponse]: """ Verify that the language set for a channel is present in at least one of its resources. @@ -818,14 +820,14 @@ def get_languages_in_channel( url_path="published_data", url_name="published-data", ) - def get_published_data(self, request, pk=None) -> Union[JsonResponse, HttpResponse]: + def get_published_data(self, request, pk=None) -> Response: """ Get the published data for a channel. :param request: The request object :param pk: The ID of the channel - :return: JsonResponse with the published data of the channel - :rtype: JsonResponse + :return: Response with the published data of the channel + :rtype: Response """ # Allow exactly users with permission to edit the channel to # access the published data. From 53c119f554e362a7a26379e8815679bd42194237 Mon Sep 17 00:00:00 2001 From: Jakub Komarek Date: Tue, 15 Jul 2025 19:30:01 +0200 Subject: [PATCH 016/472] Add action to resolve Community Library Submissions --- .../constants/community_library_submission.py | 16 + ...communitylibrarysubmission_admin_fields.py | 74 ++++ contentcuration/contentcuration/models.py | 15 + .../test_community_library_submission.py | 384 ++++++++++++++++++ contentcuration/contentcuration/urls.py | 8 + .../viewsets/community_library_submission.py | 132 ++++++ 6 files changed, 629 insertions(+) create mode 100644 contentcuration/contentcuration/migrations/0156_communitylibrarysubmission_admin_fields.py diff --git a/contentcuration/contentcuration/constants/community_library_submission.py b/contentcuration/contentcuration/constants/community_library_submission.py index 640b804f92..4e02bc5f6e 100644 --- a/contentcuration/contentcuration/constants/community_library_submission.py +++ b/contentcuration/contentcuration/constants/community_library_submission.py @@ -1,11 +1,27 @@ STATUS_PENDING = "PENDING" STATUS_APPROVED = "APPROVED" STATUS_REJECTED = "REJECTED" +STATUS_SUPERSEDED = "SUPERSEDED" STATUS_LIVE = "LIVE" status_choices = ( (STATUS_PENDING, "Pending"), (STATUS_APPROVED, "Approved"), (STATUS_REJECTED, "Rejected"), + (STATUS_SUPERSEDED, "Superseded"), (STATUS_LIVE, "Live"), ) + +REASON_INVALID_LICENSING = "INVALID_LICENSING" +REASON_TECHNICAL_QUALITY_ASSURANCE = "TECHNICAL_QUALITY_ASSURANCE" +REASON_INVALID_METADATA = "INVALID_METADATA" +REASON_PORTABILITY_ISSUES = "PORTABILITY_ISSUES" +REASON_OTHER = "OTHER" + +resolution_reason_choices = ( + (REASON_INVALID_LICENSING, "Invalid Licensing"), + (REASON_TECHNICAL_QUALITY_ASSURANCE, "Technical Quality Assurance"), + (REASON_INVALID_METADATA, "Invalid Metadata"), + (REASON_PORTABILITY_ISSUES, "Portability Issues"), + (REASON_OTHER, "Other"), +) diff --git a/contentcuration/contentcuration/migrations/0156_communitylibrarysubmission_admin_fields.py b/contentcuration/contentcuration/migrations/0156_communitylibrarysubmission_admin_fields.py new file mode 100644 index 0000000000..e88594c47a --- /dev/null +++ b/contentcuration/contentcuration/migrations/0156_communitylibrarysubmission_admin_fields.py @@ -0,0 +1,74 @@ +# Generated by Django 3.2.24 on 2025-07-15 19:12 +import django.db.models.deletion +from django.conf import settings +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + + dependencies = [ + ( + "contentcuration", + "0155_communitylibrarysubmission_submission_date_created_idx", + ), + ] + + operations = [ + migrations.AddField( + model_name="communitylibrarysubmission", + name="date_resolved", + field=models.DateTimeField(blank=True, null=True), + ), + migrations.AddField( + model_name="communitylibrarysubmission", + name="feedback_notes", + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name="communitylibrarysubmission", + name="internal_notes", + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name="communitylibrarysubmission", + name="resolution_reason", + field=models.CharField( + choices=[ + ("INVALID_LICENSING", "Invalid Licensing"), + ("TECHNICAL_QUALITY_ASSURANCE", "Technical Quality Assurance"), + ("INVALID_METADATA", "Invalid Metadata"), + ("PORTABILITY_ISSUES", "Portability Issues"), + ("OTHER", "Other"), + ], + max_length=50, + null=True, + ), + ), + migrations.AddField( + model_name="communitylibrarysubmission", + name="resolved_by", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="resolved_community_library_submissions", + to=settings.AUTH_USER_MODEL, + ), + ), + migrations.AlterField( + model_name="communitylibrarysubmission", + name="status", + field=models.CharField( + choices=[ + ("PENDING", "Pending"), + ("APPROVED", "Approved"), + ("REJECTED", "Rejected"), + ("SUPERSEDED", "Superseded"), + ("LIVE", "Live"), + ], + default="PENDING", + max_length=20, + ), + ), + ] diff --git a/contentcuration/contentcuration/models.py b/contentcuration/contentcuration/models.py index d174261fa8..f0c7305962 100644 --- a/contentcuration/contentcuration/models.py +++ b/contentcuration/contentcuration/models.py @@ -2558,11 +2558,26 @@ class CommunityLibrarySubmission(models.Model): ) categories = models.JSONField(blank=True, null=True) date_created = models.DateTimeField(auto_now_add=True) + date_resolved = models.DateTimeField(blank=True, null=True) status = models.CharField( max_length=20, choices=community_library_submission.status_choices, default=community_library_submission.STATUS_PENDING, ) + resolved_by = models.ForeignKey( + User, + related_name="resolved_community_library_submissions", + blank=True, + null=True, + on_delete=models.SET_NULL, + ) + resolution_reason = models.CharField( + max_length=50, + choices=community_library_submission.resolution_reason_choices, + null=True, + ) + feedback_notes = models.TextField(blank=True, null=True) + internal_notes = models.TextField(blank=True, null=True) def save(self, *args, **kwargs): # Validate on save that the submission author is an editor of the channel diff --git a/contentcuration/contentcuration/tests/viewsets/test_community_library_submission.py b/contentcuration/contentcuration/tests/viewsets/test_community_library_submission.py index d4de9e7d6b..cf21d9534a 100644 --- a/contentcuration/contentcuration/tests/viewsets/test_community_library_submission.py +++ b/contentcuration/contentcuration/tests/viewsets/test_community_library_submission.py @@ -1,7 +1,13 @@ +import datetime +from unittest import mock from urllib.parse import urlencode +import pytz from django.urls import reverse +from contentcuration.constants import ( + community_library_submission as community_library_submission_constants, +) from contentcuration.models import CommunityLibrarySubmission from contentcuration.tests import testdata from contentcuration.tests.base import StudioAPITestCase @@ -422,3 +428,381 @@ def test_delete_submission__is_forbidden(self): id=self.existing_submission1.id ).exists() ) + + +class AdminViewSetTestCase(StudioAPITestCase): + def setUp(self): + super().setUp() + + self.submission = testdata.community_library_submission() + self.submission.channel.version = 3 + self.submission.channel.save() + self.submission.channel_version = 2 + self.submission.save() + + self.editor_user = self.submission.channel.editors.first() + + self.superseded_submission = CommunityLibrarySubmission.objects.create( + channel=self.submission.channel, + author=self.editor_user, + status=community_library_submission_constants.STATUS_PENDING, + date_created=datetime.datetime(2023, 1, 1, tzinfo=pytz.utc), + channel_version=1, + ) + self.not_superseded_submission = CommunityLibrarySubmission.objects.create( + channel=self.submission.channel, + author=self.editor_user, + status=community_library_submission_constants.STATUS_PENDING, + date_created=datetime.datetime(2024, 1, 1, tzinfo=pytz.utc), + channel_version=3, + ) + self.submission_for_other_channel = testdata.community_library_submission() + self.submission_for_other_channel.channel_version = 1 + self.submission_for_other_channel.save() + + self.feedback_notes = "Feedback" + self.internal_notes = "Internal notes" + + self.resolve_approve_metadata = { + "status": community_library_submission_constants.STATUS_APPROVED, + "feedback_notes": self.feedback_notes, + "internal_notes": self.internal_notes, + } + self.resolve_reject_metadata = { + "status": community_library_submission_constants.STATUS_REJECTED, + "resolution_reason": community_library_submission_constants.REASON_INVALID_METADATA, + "feedback_notes": self.feedback_notes, + "internal_notes": self.internal_notes, + } + + self.resolved_time = datetime.datetime(2023, 10, 1, tzinfo=pytz.utc) + self.patcher = mock.patch( + "contentcuration.viewsets.community_library_submission.timezoned_datetime_now", + return_value=self.resolved_time, + ) + self.mock_datetime = self.patcher.start() + + def tearDown(self): + self.patcher.stop() + super().tearDown() + + def _manually_resolve_submission(self): + self.submission.status = community_library_submission_constants.STATUS_REJECTED + self.submission.resolved_by = self.admin_user + self.submission.resolution_reason = ( + community_library_submission_constants.REASON_INVALID_METADATA + ) + self.submission.feedback_notes = self.feedback_notes + self.submission.internal_notes = self.internal_notes + self.submission.date_resolved = self.resolved_time + self.submission.save() + + def _refresh_submissions_from_db(self): + self.submission.refresh_from_db() + self.superseded_submission.refresh_from_db() + self.not_superseded_submission.refresh_from_db() + self.submission_for_other_channel.refresh_from_db() + + def test_list_submissions__admin(self): + self.client.force_authenticate(user=self.admin_user) + + self._manually_resolve_submission() + + response = self.client.get( + reverse("community-library-submission-admin-list"), + ) + self.assertEqual(response.status_code, 200, response.content) + + results = response.data + self.assertEqual(len(results), 4) + rejected_results = [ + result + for result in results + if result["status"] + == community_library_submission_constants.STATUS_REJECTED + ] + self.assertEqual(len(rejected_results), 1) + result = rejected_results[0] + + self.assertEqual(result["resolved_by_id"], self.admin_user.id) + self.assertEqual(result["resolved_by_first_name"], self.admin_user.first_name) + self.assertEqual(result["resolved_by_last_name"], self.admin_user.last_name) + self.assertEqual(result["internal_notes"], self.internal_notes) + + def test_list_submissions__editor(self): + self.client.force_authenticate(user=self.editor_user) + + self._manually_resolve_submission() + + response = self.client.get( + reverse("community-library-submission-admin-list"), + ) + self.assertEqual(response.status_code, 403, response.content) + + def test_submission_detail__admin(self): + self.client.force_authenticate(user=self.admin_user) + + self._manually_resolve_submission() + + response = self.client.get( + reverse( + "community-library-submission-admin-detail", + args=[self.submission.id], + ), + ) + self.assertEqual(response.status_code, 200, response.content) + + result = response.data + self.assertEqual(result["id"], self.submission.id) + self.assertEqual(result["resolved_by_id"], self.admin_user.id) + self.assertEqual(result["resolved_by_first_name"], self.admin_user.first_name) + self.assertEqual(result["resolved_by_last_name"], self.admin_user.last_name) + self.assertEqual(result["internal_notes"], self.internal_notes) + + def test_submission_detail__editor(self): + self.client.force_authenticate(user=self.editor_user) + + self._manually_resolve_submission() + + response = self.client.get( + reverse( + "community-library-submission-admin-detail", + args=[self.submission.id], + ), + ) + self.assertEqual(response.status_code, 403, response.content) + + def test_update_submission(self): + self.client.force_authenticate(user=self.admin_user) + + response = self.client.put( + reverse( + "community-library-submission-admin-detail", + args=[self.submission.id], + ), + {}, + format="json", + ) + self.assertEqual(response.status_code, 405, response.content) + + def test_partial_update_submission(self): + self.client.force_authenticate(user=self.admin_user) + + response = self.client.patch( + reverse( + "community-library-submission-admin-detail", + args=[self.submission.id], + ), + {}, + format="json", + ) + self.assertEqual(response.status_code, 405, response.content) + + def test_destroy_submission(self): + self.client.force_authenticate(user=self.admin_user) + + response = self.client.delete( + reverse( + "community-library-submission-admin-detail", + args=[self.submission.id], + ), + format="json", + ) + self.assertEqual(response.status_code, 405, response.content) + + def test_resolve_submission__editor(self): + self.client.force_authenticate(user=self.editor_user) + response = self.client.post( + reverse( + "community-library-submission-admin-resolve", + args=[self.submission.id], + ), + self.resolve_approve_metadata, + format="json", + ) + self.assertEqual(response.status_code, 403, response.content) + + def test_resolve_submission__accept_correct(self): + self.client.force_authenticate(user=self.admin_user) + response = self.client.post( + reverse( + "community-library-submission-admin-resolve", + args=[self.submission.id], + ), + self.resolve_approve_metadata, + format="json", + ) + self.assertEqual(response.status_code, 200, response.content) + + resolved_submission = CommunityLibrarySubmission.objects.get( + id=self.submission.id + ) + self.assertEqual( + resolved_submission.status, + community_library_submission_constants.STATUS_APPROVED, + ) + self.assertEqual(resolved_submission.feedback_notes, self.feedback_notes) + self.assertEqual(resolved_submission.internal_notes, self.internal_notes) + self.assertEqual(resolved_submission.resolved_by, self.admin_user) + self.assertEqual(resolved_submission.date_resolved, self.resolved_time) + + def test_resolve_submission__reject_correct(self): + self.client.force_authenticate(user=self.admin_user) + response = self.client.post( + reverse( + "community-library-submission-admin-resolve", + args=[self.submission.id], + ), + self.resolve_reject_metadata, + format="json", + ) + self.assertEqual(response.status_code, 200, response.content) + + resolved_submission = CommunityLibrarySubmission.objects.get( + id=self.submission.id + ) + self.assertEqual( + resolved_submission.status, + community_library_submission_constants.STATUS_REJECTED, + ) + self.assertEqual( + resolved_submission.resolution_reason, + community_library_submission_constants.REASON_INVALID_METADATA, + ) + self.assertEqual(resolved_submission.feedback_notes, self.feedback_notes) + self.assertEqual(resolved_submission.internal_notes, self.internal_notes) + self.assertEqual(resolved_submission.resolved_by, self.admin_user) + self.assertEqual(resolved_submission.date_resolved, self.resolved_time) + + def test_resolve_submission__reject_missing_resolution_reason(self): + self.client.force_authenticate(user=self.admin_user) + metadata = self.resolve_reject_metadata.copy() + del metadata["resolution_reason"] + response = self.client.post( + reverse( + "community-library-submission-admin-resolve", + args=[self.submission.id], + ), + metadata, + format="json", + ) + self.assertEqual(response.status_code, 400, response.content) + + def test_resolve_submission__reject_missing_feedback_notes(self): + self.client.force_authenticate(user=self.admin_user) + metadata = self.resolve_reject_metadata.copy() + del metadata["feedback_notes"] + response = self.client.post( + reverse( + "community-library-submission-admin-resolve", + args=[self.submission.id], + ), + metadata, + format="json", + ) + self.assertEqual(response.status_code, 400, response.content) + + def test_resolve_submission__invalid_status(self): + self.client.force_authenticate(user=self.admin_user) + metadata = self.resolve_approve_metadata.copy() + metadata["status"] = (community_library_submission_constants.STATUS_PENDING,) + response = self.client.post( + reverse( + "community-library-submission-admin-resolve", + args=[self.submission.id], + ), + metadata, + format="json", + ) + self.assertEqual(response.status_code, 400, response.content) + + def test_resolve_submission__not_pending(self): + self.client.force_authenticate(user=self.admin_user) + self.submission.status = community_library_submission_constants.STATUS_APPROVED + self.submission.save() + + response = self.client.post( + reverse( + "community-library-submission-admin-resolve", + args=[self.submission.id], + ), + self.resolve_approve_metadata, + format="json", + ) + self.assertEqual(response.status_code, 400, response.content) + + def test_resolve_submission__overrite_categories(self): + self.client.force_authenticate(user=self.admin_user) + categories = ["Category 1"] + self.resolve_approve_metadata["categories"] = categories + + response = self.client.post( + reverse( + "community-library-submission-admin-resolve", + args=[self.submission.id], + ), + self.resolve_approve_metadata, + format="json", + ) + self.assertEqual(response.status_code, 200, response.content) + + resolved_submission = CommunityLibrarySubmission.objects.get( + id=self.submission.id + ) + self.assertListEqual(resolved_submission.categories, categories) + + def test_resolve_submission__accept_mark_superseded(self): + self.client.force_authenticate(user=self.admin_user) + + response = self.client.post( + reverse( + "community-library-submission-admin-resolve", + args=[self.submission.id], + ), + self.resolve_approve_metadata, + format="json", + ) + self.assertEqual(response.status_code, 200, response.content) + + self._refresh_submissions_from_db() + + self.assertEqual( + self.superseded_submission.status, + community_library_submission_constants.STATUS_SUPERSEDED, + ) + self.assertEqual( + self.not_superseded_submission.status, + community_library_submission_constants.STATUS_PENDING, + ) + self.assertEqual( + self.submission_for_other_channel.status, + community_library_submission_constants.STATUS_PENDING, + ) + + def test_resolve_submission__reject_do_not_mark_superseded(self): + self.client.force_authenticate(user=self.admin_user) + + response = self.client.post( + reverse( + "community-library-submission-admin-resolve", + args=[self.submission.id], + ), + self.resolve_reject_metadata, + format="json", + ) + self.assertEqual(response.status_code, 200, response.content) + + self._refresh_submissions_from_db() + + self.assertEqual( + self.superseded_submission.status, + community_library_submission_constants.STATUS_PENDING, + ) + self.assertEqual( + self.not_superseded_submission.status, + community_library_submission_constants.STATUS_PENDING, + ) + self.assertEqual( + self.submission_for_other_channel.status, + community_library_submission_constants.STATUS_PENDING, + ) diff --git a/contentcuration/contentcuration/urls.py b/contentcuration/contentcuration/urls.py index b948effc8b..7216eb972d 100644 --- a/contentcuration/contentcuration/urls.py +++ b/contentcuration/contentcuration/urls.py @@ -38,6 +38,9 @@ from contentcuration.viewsets.channel import ChannelViewSet from contentcuration.viewsets.channelset import ChannelSetViewSet from contentcuration.viewsets.clipboard import ClipboardViewSet +from contentcuration.viewsets.community_library_submission import ( + CommunityLibrarySubmissionAdminViewSet, +) from contentcuration.viewsets.community_library_submission import ( CommunityLibrarySubmissionViewSet, ) @@ -88,6 +91,11 @@ def get_redirect_url(self, *args, **kwargs): CommunityLibrarySubmissionViewSet, basename="community-library-submission", ) +router.register( + r"communitylibrary_submission_admin", + CommunityLibrarySubmissionAdminViewSet, + basename="community-library-submission-admin", +) urlpatterns = [ re_path(r"^api/", include(router.urls)), diff --git a/contentcuration/contentcuration/viewsets/community_library_submission.py b/contentcuration/contentcuration/viewsets/community_library_submission.py index 2aef31bfba..7610ae323e 100644 --- a/contentcuration/contentcuration/viewsets/community_library_submission.py +++ b/contentcuration/contentcuration/viewsets/community_library_submission.py @@ -1,8 +1,16 @@ +import datetime + from django_filters.rest_framework import DjangoFilterBackend +from rest_framework.decorators import action +from rest_framework.exceptions import MethodNotAllowed from rest_framework.permissions import IsAuthenticated from rest_framework.relations import PrimaryKeyRelatedField +from rest_framework.response import Response from rest_framework.serializers import ValidationError +from contentcuration.constants import ( + community_library_submission as community_library_submission_constants, +) from contentcuration.models import Channel from contentcuration.models import CommunityLibrarySubmission from contentcuration.models import Country @@ -14,6 +22,7 @@ from contentcuration.viewsets.base import RESTDestroyModelMixin from contentcuration.viewsets.base import RESTUpdateModelMixin from contentcuration.viewsets.common import UserFilteredPrimaryKeyRelatedField +from contentcuration.viewsets.user import IsAdminUser class CommunityLibrarySubmissionSerializer(BulkModelSerializer): @@ -82,6 +91,62 @@ def update(self, instance, validated_data): return super().update(instance, validated_data) +def timezoned_datetime_now(): + """ + Simple wrapper around datetime.datetime.now. The point of using a wrapper + is that mocking datetime.datetime in tests runs into issues with + isinstance in the serializer, and it is a lot easier to just mock this + wrapper. + """ + return datetime.datetime.now(datetime.timezone.utc) + + +class CommunityLibrarySubmissionResolveSerializer(CommunityLibrarySubmissionSerializer): + class Meta(CommunityLibrarySubmissionSerializer.Meta): + fields = CommunityLibrarySubmissionSerializer.Meta.fields + [ + "status", + "resolution_reason", + "feedback_notes", + "internal_notes", + ] + + def create(self, validated_data): + raise ValidationError( + "Cannot create a community library submission with this serializer. " + "Use the standard CommunityLibrarySubmissionSerializer instead." + ) + + def update(self, instance, validated_data): + if instance.status != community_library_submission_constants.STATUS_PENDING: + raise ValidationError( + "Cannot resolve a community library submission that is not pending." + ) + + if "status" not in validated_data or validated_data["status"] not in [ + community_library_submission_constants.STATUS_APPROVED, + community_library_submission_constants.STATUS_REJECTED, + ]: + raise ValidationError( + "Status must be either APPROVED or REJECTED when resolving a submission." + ) + + if ( + "status" not in validated_data + or validated_data["status"] + == community_library_submission_constants.STATUS_REJECTED + ): + if "resolution_reason" not in validated_data: + raise ValidationError( + "Resolution reason must be provided when rejecting a submission." + ) + if "feedback_notes" not in validated_data: + raise ValidationError( + "Feedback notes must be provided when rejecting a submission." + ) + + return super().update(instance, validated_data) + + class CommunityLibrarySubmissionPagination(ValuesViewsetCursorPagination): ordering = "-date_created" page_size_query_param = "max_results" @@ -105,6 +170,9 @@ class CommunityLibrarySubmissionViewSet( "categories", "date_created", "status", + "resolution_reason", + "feedback_notes", + "date_resolved", ) field_map = { "author_first_name": "author__first_name", @@ -130,3 +198,67 @@ def consolidate(self, items, queryset): item["countries"] = countries.get(item["id"], []) return items + + +class CommunityLibrarySubmissionAdminViewSet(CommunityLibrarySubmissionViewSet): + permission_classes = [IsAdminUser] + + values = CommunityLibrarySubmissionViewSet.values + ( + "resolved_by_id", + "resolved_by__first_name", + "resolved_by__last_name", + "internal_notes", + ) + field_map = CommunityLibrarySubmissionViewSet.field_map.copy() + field_map.update( + { + "resolved_by_first_name": "resolved_by__first_name", + "resolved_by_last_name": "resolved_by__last_name", + } + ) + + def create(self, request, *args, **kwargs): + raise MethodNotAllowed( + "Cannot create a community library submission with this viewset. " + "Use the standard CommunityLibrarySubmissionViewSet instead." + ) + + def update(self, instance, *args, **kwargs): + raise MethodNotAllowed( + "Cannot update a community library submission with this viewset. " + "Use the standard CommunityLibrarySubmissionViewSet instead." + ) + + def destroy(self, instance, *args, **kwargs): + raise MethodNotAllowed( + "Cannot delete a community library submission with this viewset. " + "Use the standard CommunityLibrarySubmissionViewSet instead." + ) + + def _mark_previous_pending_submissions_as_superseded(self, submission): + CommunityLibrarySubmission.objects.filter( + status=community_library_submission_constants.STATUS_PENDING, + channel=submission.channel, + channel_version__lt=submission.channel_version, + ).update(status=community_library_submission_constants.STATUS_SUPERSEDED) + + @action( + methods=["post"], + detail=True, + serializer_class=CommunityLibrarySubmissionResolveSerializer, + ) + def resolve(self, request, pk=None): + instance = self.get_edit_object() + serializer = self.get_serializer(instance, data=request.data, partial=True) + serializer.is_valid(raise_exception=True) + + date_resolved = timezoned_datetime_now() + submission = serializer.save( + date_resolved=date_resolved, + resolved_by=request.user, + ) + + if submission.status == community_library_submission_constants.STATUS_APPROVED: + self._mark_previous_pending_submissions_as_superseded(submission) + + return Response(serializer.data) From c6d22070a5ed08d951587fa15ea68928f53597b5 Mon Sep 17 00:00:00 2001 From: Jakub Komarek Date: Tue, 15 Jul 2025 22:17:44 +0200 Subject: [PATCH 017/472] Allow resolution reason to be blank --- ...nitylibrarysubmission_resolution_reason.py | 29 +++++++++++++++++++ contentcuration/contentcuration/models.py | 1 + 2 files changed, 30 insertions(+) create mode 100644 contentcuration/contentcuration/migrations/0157_alter_communitylibrarysubmission_resolution_reason.py diff --git a/contentcuration/contentcuration/migrations/0157_alter_communitylibrarysubmission_resolution_reason.py b/contentcuration/contentcuration/migrations/0157_alter_communitylibrarysubmission_resolution_reason.py new file mode 100644 index 0000000000..09a4f7d190 --- /dev/null +++ b/contentcuration/contentcuration/migrations/0157_alter_communitylibrarysubmission_resolution_reason.py @@ -0,0 +1,29 @@ +# Generated by Django 3.2.24 on 2025-07-15 20:11 +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + + dependencies = [ + ("contentcuration", "0156_communitylibrarysubmission_admin_fields"), + ] + + operations = [ + migrations.AlterField( + model_name="communitylibrarysubmission", + name="resolution_reason", + field=models.CharField( + blank=True, + choices=[ + ("INVALID_LICENSING", "Invalid Licensing"), + ("TECHNICAL_QUALITY_ASSURANCE", "Technical Quality Assurance"), + ("INVALID_METADATA", "Invalid Metadata"), + ("PORTABILITY_ISSUES", "Portability Issues"), + ("OTHER", "Other"), + ], + max_length=50, + null=True, + ), + ), + ] diff --git a/contentcuration/contentcuration/models.py b/contentcuration/contentcuration/models.py index f0c7305962..b62441402a 100644 --- a/contentcuration/contentcuration/models.py +++ b/contentcuration/contentcuration/models.py @@ -2574,6 +2574,7 @@ class CommunityLibrarySubmission(models.Model): resolution_reason = models.CharField( max_length=50, choices=community_library_submission.resolution_reason_choices, + blank=True, null=True, ) feedback_notes = models.TextField(blank=True, null=True) From dce292bcdc1ca8bb50cea3c0a977d382c2f41dda Mon Sep 17 00:00:00 2001 From: habibayman Date: Sat, 19 Jul 2025 07:20:21 +0300 Subject: [PATCH 018/472] config: add tiptap-markdown dependency --- package.json | 1 + pnpm-lock.yaml | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/package.json b/package.json index d6f1092fb8..f261d4e2ce 100644 --- a/package.json +++ b/package.json @@ -97,6 +97,7 @@ "spark-md5": "^3.0.0", "store2": "^2.14.4", "string-strip-html": "8.3.0", + "tiptap-markdown": "^0.8.10", "uuid": "^11.1.0", "vue": "~2.7.16", "vue-croppa": "^1.3.8", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e290aa9f14..e148fb010d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -140,6 +140,9 @@ importers: string-strip-html: specifier: 8.3.0 version: 8.3.0 + tiptap-markdown: + specifier: ^0.8.10 + version: 0.8.10(@tiptap/core@2.23.1(@tiptap/pm@2.23.1)) uuid: specifier: ^11.1.0 version: 11.1.0 @@ -1698,6 +1701,9 @@ packages: '@types/json5@0.0.29': resolution: {integrity: sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==} + '@types/linkify-it@3.0.5': + resolution: {integrity: sha512-yg6E+u0/+Zjva+buc3EIb+29XEg4wltq7cSmd4Uc2EE/1nUVmxyzpX6gUXD0V8jIrG0r7YeOGVIbYRkxeooCtw==} + '@types/linkify-it@5.0.0': resolution: {integrity: sha512-sVDA58zAw4eWAffKOaQH5/5j3XeayukzDk+ewSsnv3p4yJEZHCCzMDiZM8e0OUrRvmpGZ85jf4yDHkHsgBNr9Q==} @@ -1708,9 +1714,15 @@ packages: '@types/lodash@4.17.20': resolution: {integrity: sha512-H3MHACvFUEiujabxhaI/ImO6gUrd8oOurg7LQtS7mbwIXA/cUqWrvBsaeJ23aZEPk1TAYkurjfMbSELfoCXlGA==} + '@types/markdown-it@13.0.9': + resolution: {integrity: sha512-1XPwR0+MgXLWfTn9gCsZ55AHOKW1WN+P9vr0PaQh5aerR9LLQXUbjfEAFhjmEmyoYFWAyuN2Mqkn40MZ4ukjBw==} + '@types/markdown-it@14.1.2': resolution: {integrity: sha512-promo4eFwuiW+TfGxhi+0x3czqTYJkG8qB17ZUJiVF10Xm7NLVRSLUsfRTU/6h1e24VvRnXCx+hG7li58lkzog==} + '@types/mdurl@1.0.5': + resolution: {integrity: sha512-6L6VymKTzYSrEf4Nev4Xa1LCHKrlTlYCBMTlQKFuddo1CvQcE52I0mwfOJayueUC7MJuXOeHTcIU683lzd0cUA==} + '@types/mdurl@2.0.0': resolution: {integrity: sha512-RGdgjQUZba5p6QEFAVx2OGb8rQDL/cPRG7GiedRzMcJ1tYnUANBncjbSB1NRGwbvjcPeikRABz2nshyPk1bhWg==} @@ -5039,6 +5051,9 @@ packages: map-values@1.0.1: resolution: {integrity: sha512-BbShUnr5OartXJe1GeccAWtfro11hhgNJg6G9/UtWKjVGvV5U4C09cg5nk8JUevhXODaXY+hQ3xxMUKSs62ONQ==} + markdown-it-task-lists@2.1.1: + resolution: {integrity: sha512-TxFAc76Jnhb2OUu+n3yz9RMu4CwGfaT788br6HhEDlvWfdeJcLUsxk1Hgw2yJio0OXsxv7pyIPmvECY7bMbluA==} + markdown-it@14.1.0: resolution: {integrity: sha512-a54IwgWPaeBCAAsv13YgmALOF1elABB08FxO9i+r4VFk5Vl4pKokRPeX8u5TCgSsPi6ec1otfLjdOpVcgbpshg==} hasBin: true @@ -7029,6 +7044,11 @@ packages: tippy.js@6.3.7: resolution: {integrity: sha512-E1d3oP2emgJ9dRQZdf3Kkn0qJgI6ZLpyS5z6ZkY1DF3kaQaBsGZsndEpHwx+eC+tYM41HaSNvNtLx8tU57FzTQ==} + tiptap-markdown@0.8.10: + resolution: {integrity: sha512-iDVkR2BjAqkTDtFX0h94yVvE2AihCXlF0Q7RIXSJPRSR5I0PA1TMuAg6FHFpmqTn4tPxJ0by0CK7PUMlnFLGEQ==} + peerDependencies: + '@tiptap/core': ^2.0.3 + tmp@0.2.3: resolution: {integrity: sha512-nZD7m9iCPC5g0pYmcaxogYKggSfLsdxl8of3Q/oIbqCqLLIO9IAF0GWjX1z9NZRHPiXv8Wex4yDCaZsgEw0Y8w==} engines: {node: '>=14.14'} @@ -9451,6 +9471,8 @@ snapshots: '@types/json5@0.0.29': {} + '@types/linkify-it@3.0.5': {} + '@types/linkify-it@5.0.0': {} '@types/localforage@0.0.34': @@ -9459,11 +9481,18 @@ snapshots: '@types/lodash@4.17.20': {} + '@types/markdown-it@13.0.9': + dependencies: + '@types/linkify-it': 3.0.5 + '@types/mdurl': 1.0.5 + '@types/markdown-it@14.1.2': dependencies: '@types/linkify-it': 5.0.0 '@types/mdurl': 2.0.0 + '@types/mdurl@1.0.5': {} + '@types/mdurl@2.0.0': {} '@types/mime@1.3.5': {} @@ -13569,6 +13598,8 @@ snapshots: map-values@1.0.1: {} + markdown-it-task-lists@2.1.1: {} + markdown-it@14.1.0: dependencies: argparse: 2.0.1 @@ -15742,6 +15773,14 @@ snapshots: dependencies: '@popperjs/core': 2.11.8 + tiptap-markdown@0.8.10(@tiptap/core@2.23.1(@tiptap/pm@2.23.1)): + dependencies: + '@tiptap/core': 2.23.1(@tiptap/pm@2.23.1) + '@types/markdown-it': 13.0.9 + markdown-it: 14.1.0 + markdown-it-task-lists: 2.1.1 + prosemirror-markdown: 1.13.2 + tmp@0.2.3: {} tmpl@1.0.5: {} From 4086f591535e82fd7ba96fd8c59dac059991d5ab Mon Sep 17 00:00:00 2001 From: habibayman Date: Sat, 19 Jul 2025 07:22:25 +0300 Subject: [PATCH 019/472] feat(texteditor)[markdown]: configure bidirectional conversion --- .../TipTapEditor/composables/useEditor.js | 11 ++-- .../TipTapEditor/extensions/Markdown.js | 64 +++++++++++++++++++ .../TipTapEditor/utils/markdown.js | 46 +++++++++++++ 3 files changed, 117 insertions(+), 4 deletions(-) create mode 100644 contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/extensions/Markdown.js create mode 100644 contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/utils/markdown.js diff --git a/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/composables/useEditor.js b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/composables/useEditor.js index 47b1b48501..216adb5d94 100644 --- a/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/composables/useEditor.js +++ b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/composables/useEditor.js @@ -9,12 +9,13 @@ import { Image } from '../extensions/Image'; import { CodeBlockSyntaxHighlight } from '../extensions/CodeBlockSyntaxHighlight'; import { CustomLink } from '../extensions/Link'; import { Math } from '../extensions/Math'; +import { Markdown } from '../extensions/Markdown'; export function useEditor() { const editor = ref(null); const isReady = ref(false); - const initializeEditor = () => { + const initializeEditor = content => { editor.value = new Editor({ extensions: [ StarterKitExtension.configure({ @@ -29,17 +30,19 @@ export function useEditor() { Image, CustomLink, // Use our custom Link extension Math, + Markdown, ], - content: '

', + content: content || '

', editorProps: { attributes: { class: 'prose prose-sm sm:prose lg:prose-lg xl:prose-2xl focus:outline-none', dir: 'auto', }, }, + onCreate: () => { + isReady.value = true; + }, }); - - isReady.value = true; }; const destroyEditor = () => { diff --git a/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/extensions/Markdown.js b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/extensions/Markdown.js new file mode 100644 index 0000000000..dd6b2f668d --- /dev/null +++ b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/extensions/Markdown.js @@ -0,0 +1,64 @@ +import { Markdown as TiptapMarkdown } from 'tiptap-markdown'; +import { + IMAGE_REGEX, + imageMdToParams, + paramsToImageMd, + MATH_REGEX, + mathMdToParams, + paramsToMathMd, +} from '../utils/markdown'; + +export const Markdown = TiptapMarkdown.configure({ + html: true, + bulletList: { + tight: true, + }, + orderedList: { + tight: true, + }, + // --- LOADING CONFIGURATION --- + // This hook pre-processes the raw Markdown string before parsing. + transformMarkdown: markdown => { + let processedMarkdown = markdown; + + // Replace custom images with standard tags + processedMarkdown = processedMarkdown.replace(IMAGE_REGEX, match => { + const params = imageMdToParams(match); + if (!params) return match; + return `${params.alt}`; + }); + + // Replace $$...$$ with a custom tag for our Math extension + processedMarkdown = processedMarkdown.replace(MATH_REGEX, match => { + const params = mathMdToParams(match); + if (!params) return match; + return ``; + }); + + return processedMarkdown; + }, + + // --- SAVING CONFIGURATION --- + // These rules override the default serializers for specific nodes and marks. + toMarkdown: { + // --- Custom Node Rules --- + image(state, node) { + state.write(paramsToImageMd(node.attrs)); + }, + math(state, node) { + state.write(paramsToMathMd(node.attrs)); + }, + small(state, node) { + state.write(''); + state.renderContent(node); + state.write(''); + state.closeBlock(node); + }, + // --- Custom Mark Rules --- + underline: { + open: '', + close: '', + mixable: true, + }, + }, +}); diff --git a/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/utils/markdown.js b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/utils/markdown.js new file mode 100644 index 0000000000..295fd6ba65 --- /dev/null +++ b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/utils/markdown.js @@ -0,0 +1,46 @@ +// Contains utilities for handling Markdown content bidirectional conversion in TipTap editor. + +// --- Image Translation --- +export const IMAGE_PLACEHOLDER = 'placeholder'; +export const IMAGE_REGEX = /!\[([^\]]*)\]\(([^/]+\/[^\s=)]+)(?:\s*=\s*([0-9.]+)x([0-9.]+))?\)/g; + +export const imageMdToParams = markdown => { + // Reset regex state before executing to ensure it works on all matches + IMAGE_REGEX.lastIndex = 0; + const match = IMAGE_REGEX.exec(markdown); + if (!match) return null; + + const [, alt, fullPath, width, height] = match; + const pathParts = fullPath.split('/'); + + // Ensure it matches the "placeholder/checksum.ext" structure + if (pathParts.length < 2 || pathParts[0] !== IMAGE_PLACEHOLDER) { + return null; + } + + const src = pathParts.slice(1).join('/'); // The "checksum.ext" part + + return { src, alt: alt || '', width: width || null, height: height || null }; +}; + +export const paramsToImageMd = ({ src, alt, width, height }) => { + const fileName = src.split('/').pop(); + if (width && height) { + return `![${alt || ''}](${IMAGE_PLACEHOLDER}/${fileName} =${width}x${height})`; + } + return `![${alt || ''}](${IMAGE_PLACEHOLDER}/${fileName})`; +}; + +// --- Math/Formula Translation --- +export const MATH_REGEX = /\$\$([^$]+)\$\$/g; + +export const mathMdToParams = markdown => { + MATH_REGEX.lastIndex = 0; + const match = MATH_REGEX.exec(markdown); + if (!match) return null; + return { latex: match[1].trim() }; +}; + +export const paramsToMathMd = ({ latex }) => { + return `$$${latex || ''}$$`; +}; From 52a1d8d88824bd836596411729c9be0e0e4f7b73 Mon Sep 17 00:00:00 2001 From: habibayman Date: Sat, 19 Jul 2025 07:34:51 +0300 Subject: [PATCH 020/472] feat(texteditor)[markdwon]: create temp fake parent --- .../frontend/channelEdit/router.js | 4 +- .../frontend/editorDev/index.js | 4 +- .../frontend/editorDev/router.js | 4 +- .../TipTapEditor/TipTapEditor/DevHarness.vue | 113 ++++++++++++++++++ .../TipTapEditor/TipTapEditor.vue | 79 +++++++++++- 5 files changed, 192 insertions(+), 12 deletions(-) create mode 100644 contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/DevHarness.vue diff --git a/contentcuration/contentcuration/frontend/channelEdit/router.js b/contentcuration/contentcuration/frontend/channelEdit/router.js index 3d362cbb59..b12d63857b 100644 --- a/contentcuration/contentcuration/frontend/channelEdit/router.js +++ b/contentcuration/contentcuration/frontend/channelEdit/router.js @@ -11,7 +11,7 @@ import ReviewSelectionsPage from './views/ImportFromChannels/ReviewSelectionsPag import EditModal from './components/edit/EditModal'; import ChannelDetailsModal from 'shared/views/channel/ChannelDetailsModal'; import ChannelModal from 'shared/views/channel/ChannelModal'; -import TipTapEditor from 'shared/views/TipTapEditor/TipTapEditor/TipTapEditor.vue'; +import DevHarness from 'shared/views/TipTapEditor/TipTapEditor/DevHarness.vue'; import { RouteNames as ChannelRouteNames } from 'frontend/channelList/constants'; const router = new VueRouter({ @@ -19,7 +19,7 @@ const router = new VueRouter({ { path: '/editor-dev', name: 'TipTapEditorDev', - component: TipTapEditor, + component: DevHarness, }, { name: RouteNames.TREE_ROOT_VIEW, diff --git a/contentcuration/contentcuration/frontend/editorDev/index.js b/contentcuration/contentcuration/frontend/editorDev/index.js index ef9a10c8db..4b7e6e0c63 100644 --- a/contentcuration/contentcuration/frontend/editorDev/index.js +++ b/contentcuration/contentcuration/frontend/editorDev/index.js @@ -1,7 +1,7 @@ import Vue from 'vue'; import Vuex from 'vuex'; import VueRouter from 'vue-router'; -import TipTapEditor from '../shared/views/TipTapEditor/TipTapEditor/TipTapEditor.vue'; +import DevHarness from '../shared/views/TipTapEditor/TipTapEditor/DevHarness.vue'; import startApp from 'shared/app'; import storeFactory from 'shared/vuex/baseStore'; @@ -17,7 +17,7 @@ startApp({ routes: [ { path: '/', - component: TipTapEditor, + component: DevHarness, }, ], }), diff --git a/contentcuration/contentcuration/frontend/editorDev/router.js b/contentcuration/contentcuration/frontend/editorDev/router.js index 1bb8ee6838..bdbebda15f 100644 --- a/contentcuration/contentcuration/frontend/editorDev/router.js +++ b/contentcuration/contentcuration/frontend/editorDev/router.js @@ -1,10 +1,10 @@ import Vue from 'vue'; import Router from 'vue-router'; -import TipTapEditor from '../shared/views/TipTapEditor/TipTapEditor/TipTapEditor.vue'; +import DevHarness from '../shared/views/TipTapEditor/TipTapEditor/DevHarness.vue'; Vue.use(Router); export default new Router({ mode: 'history', base: '/editor-dev/', - routes: [{ path: '/', component: TipTapEditor }], + routes: [{ path: '/', component: DevHarness }], }); diff --git a/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/DevHarness.vue b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/DevHarness.vue new file mode 100644 index 0000000000..3dff5b605f --- /dev/null +++ b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/DevHarness.vue @@ -0,0 +1,113 @@ + + + + + + + diff --git a/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/TipTapEditor.vue b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/TipTapEditor.vue index 321b40743d..3257bff77a 100644 --- a/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/TipTapEditor.vue +++ b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/TipTapEditor.vue @@ -73,7 +73,7 @@ From d0460563b8cf7e0a9c738496b52d343990183d18 Mon Sep 17 00:00:00 2001 From: habibayman Date: Sun, 20 Jul 2025 03:43:32 +0300 Subject: [PATCH 021/472] refactor(texteditor): create custom markdown serializer --- .../TipTapEditor/TipTapEditor/DevHarness.vue | 7 +- .../TipTapEditor/TipTapEditor.vue | 20 +- .../TipTapEditor/composables/useEditor.js | 6 + .../TipTapEditor/extensions/Markdown.js | 54 +---- .../TipTapEditor/utils/markdown.js | 27 +++ .../TipTapEditor/utils/markdownSerializer.js | 207 ++++++++++++++++++ 6 files changed, 248 insertions(+), 73 deletions(-) create mode 100644 contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/utils/markdownSerializer.js diff --git a/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/DevHarness.vue b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/DevHarness.vue index 3dff5b605f..c461a05641 100644 --- a/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/DevHarness.vue +++ b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/DevHarness.vue @@ -6,10 +6,7 @@

This page simulates a parent component to test the editor in isolation.

- +

Live Markdown Output (v-model state)

@@ -29,7 +26,7 @@ **bold** *italic* underline ~~strikethrough~~ -try inline formulas test +try inline formulas $$x^2$$ test - list a - list b diff --git a/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/TipTapEditor.vue b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/TipTapEditor.vue index 3257bff77a..5fa36f768a 100644 --- a/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/TipTapEditor.vue +++ b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/TipTapEditor.vue @@ -85,6 +85,7 @@ import LinkEditor from './components/link/LinkEditor.vue'; import { useMathHandling } from './composables/useMathHandling'; import FormulasMenu from './components/math/FormulasMenu.vue'; + import { preprocessMarkdown } from './utils/markdown'; export default defineComponent({ name: 'RichTextEditor', @@ -133,33 +134,23 @@ return editor.value.storage.markdown.getMarkdown(); }; - const setMarkdownContent = (content, emitUpdate = false) => { - if (!editor.value || !isReady.value || !editor.value.storage?.markdown) { - return; - } - editor.value.storage.markdown.setMarkdown(content, emitUpdate); - }; - let isUpdatingFromOutside = false; // A flag to prevent infinite update loops // sync changes from the parent component to the editor watch( () => props.value, newValue => { - if (!editor.value) { - initializeEditor(newValue); - return; - } + const processedContent = preprocessMarkdown(newValue); - if (!isReady.value || !editor.value.storage?.markdown) { + if (!editor.value) { + initializeEditor(processedContent); return; } const editorContent = getMarkdownContent(); - if (editorContent !== newValue) { isUpdatingFromOutside = true; - setMarkdownContent(newValue, false); + editor.value.commands.setContent(processedContent, false); nextTick(() => { isUpdatingFromOutside = false; }); @@ -182,7 +173,6 @@ } const markdown = getMarkdownContent(); - if (markdown !== props.value) { emit('input', markdown); } diff --git a/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/composables/useEditor.js b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/composables/useEditor.js index 216adb5d94..d30dcc5207 100644 --- a/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/composables/useEditor.js +++ b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/composables/useEditor.js @@ -10,6 +10,7 @@ import { CodeBlockSyntaxHighlight } from '../extensions/CodeBlockSyntaxHighlight import { CustomLink } from '../extensions/Link'; import { Math } from '../extensions/Math'; import { Markdown } from '../extensions/Markdown'; +import { createCustomMarkdownSerializer } from '../utils/markdownSerializer'; export function useEditor() { const editor = ref(null); @@ -41,6 +42,11 @@ export function useEditor() { }, onCreate: () => { isReady.value = true; + + const markdownStorage = editor.value.storage.markdown; + if (markdownStorage) { + markdownStorage.getMarkdown = createCustomMarkdownSerializer(editor.value); + } }, }); }; diff --git a/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/extensions/Markdown.js b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/extensions/Markdown.js index dd6b2f668d..25d16bcb21 100644 --- a/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/extensions/Markdown.js +++ b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/extensions/Markdown.js @@ -1,13 +1,6 @@ import { Markdown as TiptapMarkdown } from 'tiptap-markdown'; -import { - IMAGE_REGEX, - imageMdToParams, - paramsToImageMd, - MATH_REGEX, - mathMdToParams, - paramsToMathMd, -} from '../utils/markdown'; +// Minimal configuration - we handle preprocessing manually via preprocessMarkdown() export const Markdown = TiptapMarkdown.configure({ html: true, bulletList: { @@ -16,49 +9,4 @@ export const Markdown = TiptapMarkdown.configure({ orderedList: { tight: true, }, - // --- LOADING CONFIGURATION --- - // This hook pre-processes the raw Markdown string before parsing. - transformMarkdown: markdown => { - let processedMarkdown = markdown; - - // Replace custom images with standard tags - processedMarkdown = processedMarkdown.replace(IMAGE_REGEX, match => { - const params = imageMdToParams(match); - if (!params) return match; - return `${params.alt}`; - }); - - // Replace $$...$$ with a custom tag for our Math extension - processedMarkdown = processedMarkdown.replace(MATH_REGEX, match => { - const params = mathMdToParams(match); - if (!params) return match; - return ``; - }); - - return processedMarkdown; - }, - - // --- SAVING CONFIGURATION --- - // These rules override the default serializers for specific nodes and marks. - toMarkdown: { - // --- Custom Node Rules --- - image(state, node) { - state.write(paramsToImageMd(node.attrs)); - }, - math(state, node) { - state.write(paramsToMathMd(node.attrs)); - }, - small(state, node) { - state.write(''); - state.renderContent(node); - state.write(''); - state.closeBlock(node); - }, - // --- Custom Mark Rules --- - underline: { - open: '', - close: '', - mixable: true, - }, - }, }); diff --git a/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/utils/markdown.js b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/utils/markdown.js index 295fd6ba65..62fb18d3a0 100644 --- a/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/utils/markdown.js +++ b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/utils/markdown.js @@ -44,3 +44,30 @@ export const mathMdToParams = markdown => { export const paramsToMathMd = ({ latex }) => { return `$$${latex || ''}$$`; }; + +/** + * Pre-processes a raw Markdown string to convert custom syntax into HTML tags + * that Tiptap's extensions can understand. This is our custom "loader". + * @param {string} markdown - The raw markdown string. + * @returns {string} - The processed string with HTML tags. + */ +export function preprocessMarkdown(markdown) { + if (!markdown) return ''; + let processedMarkdown = markdown; + + // Replace custom images with standard tags + processedMarkdown = processedMarkdown.replace(IMAGE_REGEX, match => { + const params = imageMdToParams(match); + if (!params) return match; + return `${params.alt}`; + }); + + // Replace $$...$$ with a custom tag for our Math extension + processedMarkdown = processedMarkdown.replace(MATH_REGEX, match => { + const params = mathMdToParams(match); + if (!params) return match; + return ``; + }); + + return processedMarkdown; +} diff --git a/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/utils/markdownSerializer.js b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/utils/markdownSerializer.js new file mode 100644 index 0000000000..261a59372e --- /dev/null +++ b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/utils/markdownSerializer.js @@ -0,0 +1,207 @@ +// Custom markdown serialization that handles Math nodes properly +import { paramsToMathMd, paramsToImageMd } from './markdown'; + +export const createCustomMarkdownSerializer = editor => { + return function getMarkdown() { + const doc = editor.state.doc; + let result = ''; + + // Handle marks (bold, italic, etc.) + const serializeMarks = node => { + if (!node.marks || node.marks.length === 0) return node.text || ''; + + let text = node.text || ''; + node.marks.forEach(mark => { + switch (mark.type.name) { + case 'bold': + text = `**${text}**`; + break; + case 'italic': + text = `*${text}*`; + break; + case 'underline': + text = `${text}`; + break; + case 'strike': + text = `~~${text}~~`; + break; + case 'code': + text = `\`${text}\``; + break; + case 'link': { + const href = mark.attrs.href || ''; + text = `[${text}](${href})`; + break; + } + case 'superscript': + text = `${text}`; + break; + case 'subscript': + text = `${text}`; + break; + } + }); + return text; + }; + + const serializeNode = (node, listNumber = null) => { + if (!node || !node.type) { + return; + } + + switch (node.type.name) { + case 'doc': + // Process all children + if (node.content && node.content.size > 0) { + for (let i = 0; i < node.content.size; i++) { + const child = node.content.content[i]; + if (child) { + if (i > 0) result += '\n\n'; + serializeNode(child); + } + } + } + break; + + case 'paragraph': + if (node.content && node.content.size > 0) { + for (let i = 0; i < node.content.size; i++) { + const child = node.content.content[i]; + if (child) { + serializeNode(child); + } + } + } + break; + + case 'heading': { + const level = node.attrs.level || 1; + result += '#'.repeat(level) + ' '; + if (node.content && node.content.size > 0) { + for (let i = 0; i < node.content.size; i++) { + const child = node.content.content[i]; + if (child) { + serializeNode(child); + } + } + } + break; + } + + case 'text': + result += serializeMarks(node); + break; + + case 'math': + result += paramsToMathMd(node.attrs); + break; + + case 'image': + result += paramsToImageMd(node.attrs); + break; + + case 'small': + result += ''; + if (node.content && node.content.size > 0) { + for (let i = 0; i < node.content.size; i++) { + const child = node.content.content[i]; + if (child) { + serializeNode(child); + } + } + } + result += ''; + break; + + case 'bulletList': + for (let i = 0; i < node.content.size; i++) { + const child = node.content.content[i]; + if (child) { + serializeNode(child, 'bullet'); + if (i < node.content.size - 1) result += '\n'; + } + } + break; + + case 'orderedList': + for (let i = 0; i < node.content.size; i++) { + const child = node.content.content[i]; + if (child) { + serializeNode(child, i + 1); + if (i < node.content.size - 1) result += '\n'; + } + } + break; + + case 'listItem': + // Use the passed listNumber parameter + if (listNumber === 'bullet') { + result += '- '; + } else if (typeof listNumber === 'number') { + result += `${listNumber}. `; + } + + // Process list item content properly + if (node.content && node.content.size > 0) { + for (let i = 0; i < node.content.size; i++) { + const child = node.content.content[i]; + if (child && child.type) { + if (child.type.name === 'paragraph') { + // For paragraphs in list items, process their content directly + if (child.content && child.content.size > 0) { + for (let j = 0; j < child.content.size; j++) { + const grandchild = child.content.content[j]; + if (grandchild) { + serializeNode(grandchild); + } + } + } + } else { + serializeNode(child); + } + } + } + } + break; + + case 'blockquote': + result += '> '; + if (node.content && node.content.size > 0) { + for (let i = 0; i < node.content.size; i++) { + const child = node.content.content[i]; + if (child) { + serializeNode(child); + } + } + } + break; + + case 'codeBlock': { + const language = node.attrs.language || ''; + result += '```' + language + '\n'; + result += node.textContent; + result += '\n```'; + break; + } + + case 'hardBreak': + result += ' \n'; + break; + + case 'horizontalRule': + result += '---'; + break; + + default: + // Fallback: try to process children + if (node.content) { + node.content.forEach(child => serializeNode(child)); + } + break; + } + }; + + serializeNode(doc, true); + return result.trim(); + }; +}; From 4788ad2dc00d9de6a7641ed1669b522e75691b1b Mon Sep 17 00:00:00 2001 From: habibayman Date: Sun, 20 Jul 2025 05:02:20 +0300 Subject: [PATCH 022/472] refactor(texteditor): remove tiptap-markdown with related logic --- .../TipTapEditor/TipTapEditor/DevHarness.vue | 2 +- .../TipTapEditor/composables/useEditor.js | 9 ++-- .../TipTapEditor/extensions/Markdown.js | 12 ----- .../TipTapEditor/utils/markdown.js | 9 ++-- package.json | 2 +- pnpm-lock.yaml | 49 ++++--------------- 6 files changed, 22 insertions(+), 61 deletions(-) delete mode 100644 contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/extensions/Markdown.js diff --git a/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/DevHarness.vue b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/DevHarness.vue index c461a05641..9ba867b339 100644 --- a/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/DevHarness.vue +++ b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/DevHarness.vue @@ -35,7 +35,7 @@ try inline formulas $$x^2$$ test small text -1. list one +1. list one[1] 2. list two There is a [link here](https://github.com/learningequality/studio/pull/5155/checks)! diff --git a/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/composables/useEditor.js b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/composables/useEditor.js index d30dcc5207..fd400779b0 100644 --- a/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/composables/useEditor.js +++ b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/composables/useEditor.js @@ -9,7 +9,6 @@ import { Image } from '../extensions/Image'; import { CodeBlockSyntaxHighlight } from '../extensions/CodeBlockSyntaxHighlight'; import { CustomLink } from '../extensions/Link'; import { Math } from '../extensions/Math'; -import { Markdown } from '../extensions/Markdown'; import { createCustomMarkdownSerializer } from '../utils/markdownSerializer'; export function useEditor() { @@ -31,7 +30,6 @@ export function useEditor() { Image, CustomLink, // Use our custom Link extension Math, - Markdown, ], content: content || '

', editorProps: { @@ -43,10 +41,11 @@ export function useEditor() { onCreate: () => { isReady.value = true; - const markdownStorage = editor.value.storage.markdown; - if (markdownStorage) { - markdownStorage.getMarkdown = createCustomMarkdownSerializer(editor.value); + // Create a simple storage object to hold our custom markdown serializer + if (!editor.value.storage.markdown) { + editor.value.storage.markdown = {}; } + editor.value.storage.markdown.getMarkdown = createCustomMarkdownSerializer(editor.value); }, }); }; diff --git a/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/extensions/Markdown.js b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/extensions/Markdown.js deleted file mode 100644 index 25d16bcb21..0000000000 --- a/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/extensions/Markdown.js +++ /dev/null @@ -1,12 +0,0 @@ -import { Markdown as TiptapMarkdown } from 'tiptap-markdown'; - -// Minimal configuration - we handle preprocessing manually via preprocessMarkdown() -export const Markdown = TiptapMarkdown.configure({ - html: true, - bulletList: { - tight: true, - }, - orderedList: { - tight: true, - }, -}); diff --git a/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/utils/markdown.js b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/utils/markdown.js index 62fb18d3a0..38227c95d1 100644 --- a/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/utils/markdown.js +++ b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/utils/markdown.js @@ -1,4 +1,6 @@ // Contains utilities for handling Markdown content bidirectional conversion in TipTap editor. +// eslint-disable-next-line import/namespace +import { marked } from 'marked'; // --- Image Translation --- export const IMAGE_PLACEHOLDER = 'placeholder'; @@ -53,21 +55,22 @@ export const paramsToMathMd = ({ latex }) => { */ export function preprocessMarkdown(markdown) { if (!markdown) return ''; + let processedMarkdown = markdown; - // Replace custom images with standard tags + // First handle your custom syntax (images and math) as before processedMarkdown = processedMarkdown.replace(IMAGE_REGEX, match => { const params = imageMdToParams(match); if (!params) return match; return `${params.alt}`; }); - // Replace $$...$$ with a custom tag for our Math extension processedMarkdown = processedMarkdown.replace(MATH_REGEX, match => { const params = mathMdToParams(match); if (!params) return match; return ``; }); - return processedMarkdown; + // Use marked.js to parse the rest of the markdown + return marked(processedMarkdown); } diff --git a/package.json b/package.json index f261d4e2ce..c1c81ffdbf 100644 --- a/package.json +++ b/package.json @@ -84,6 +84,7 @@ "kolibri-design-system": "5.2.0", "lodash": "^4.17.21", "lowlight": "^3.3.0", + "marked": "^16.1.1", "material-icons": "0.3.1", "mathlive": "^0.105.3", "mutex-js": "^1.1.5", @@ -97,7 +98,6 @@ "spark-md5": "^3.0.0", "store2": "^2.14.4", "string-strip-html": "8.3.0", - "tiptap-markdown": "^0.8.10", "uuid": "^11.1.0", "vue": "~2.7.16", "vue-croppa": "^1.3.8", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e148fb010d..4eb3e61e65 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -101,6 +101,9 @@ importers: lowlight: specifier: ^3.3.0 version: 3.3.0 + marked: + specifier: ^16.1.1 + version: 16.1.1 material-icons: specifier: 0.3.1 version: 0.3.1 @@ -140,9 +143,6 @@ importers: string-strip-html: specifier: 8.3.0 version: 8.3.0 - tiptap-markdown: - specifier: ^0.8.10 - version: 0.8.10(@tiptap/core@2.23.1(@tiptap/pm@2.23.1)) uuid: specifier: ^11.1.0 version: 11.1.0 @@ -1701,9 +1701,6 @@ packages: '@types/json5@0.0.29': resolution: {integrity: sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==} - '@types/linkify-it@3.0.5': - resolution: {integrity: sha512-yg6E+u0/+Zjva+buc3EIb+29XEg4wltq7cSmd4Uc2EE/1nUVmxyzpX6gUXD0V8jIrG0r7YeOGVIbYRkxeooCtw==} - '@types/linkify-it@5.0.0': resolution: {integrity: sha512-sVDA58zAw4eWAffKOaQH5/5j3XeayukzDk+ewSsnv3p4yJEZHCCzMDiZM8e0OUrRvmpGZ85jf4yDHkHsgBNr9Q==} @@ -1714,15 +1711,9 @@ packages: '@types/lodash@4.17.20': resolution: {integrity: sha512-H3MHACvFUEiujabxhaI/ImO6gUrd8oOurg7LQtS7mbwIXA/cUqWrvBsaeJ23aZEPk1TAYkurjfMbSELfoCXlGA==} - '@types/markdown-it@13.0.9': - resolution: {integrity: sha512-1XPwR0+MgXLWfTn9gCsZ55AHOKW1WN+P9vr0PaQh5aerR9LLQXUbjfEAFhjmEmyoYFWAyuN2Mqkn40MZ4ukjBw==} - '@types/markdown-it@14.1.2': resolution: {integrity: sha512-promo4eFwuiW+TfGxhi+0x3czqTYJkG8qB17ZUJiVF10Xm7NLVRSLUsfRTU/6h1e24VvRnXCx+hG7li58lkzog==} - '@types/mdurl@1.0.5': - resolution: {integrity: sha512-6L6VymKTzYSrEf4Nev4Xa1LCHKrlTlYCBMTlQKFuddo1CvQcE52I0mwfOJayueUC7MJuXOeHTcIU683lzd0cUA==} - '@types/mdurl@2.0.0': resolution: {integrity: sha512-RGdgjQUZba5p6QEFAVx2OGb8rQDL/cPRG7GiedRzMcJ1tYnUANBncjbSB1NRGwbvjcPeikRABz2nshyPk1bhWg==} @@ -5051,13 +5042,15 @@ packages: map-values@1.0.1: resolution: {integrity: sha512-BbShUnr5OartXJe1GeccAWtfro11hhgNJg6G9/UtWKjVGvV5U4C09cg5nk8JUevhXODaXY+hQ3xxMUKSs62ONQ==} - markdown-it-task-lists@2.1.1: - resolution: {integrity: sha512-TxFAc76Jnhb2OUu+n3yz9RMu4CwGfaT788br6HhEDlvWfdeJcLUsxk1Hgw2yJio0OXsxv7pyIPmvECY7bMbluA==} - markdown-it@14.1.0: resolution: {integrity: sha512-a54IwgWPaeBCAAsv13YgmALOF1elABB08FxO9i+r4VFk5Vl4pKokRPeX8u5TCgSsPi6ec1otfLjdOpVcgbpshg==} hasBin: true + marked@16.1.1: + resolution: {integrity: sha512-ij/2lXfCRT71L6u0M29tJPhP0bM5shLL3u5BePhFwPELj2blMJ6GDtD7PfJhRLhJ/c2UwrK17ySVcDzy2YHjHQ==} + engines: {node: '>= 20'} + hasBin: true + marks-pane@1.0.9: resolution: {integrity: sha512-Ahs4oeG90tbdPWwAJkAAoHg2lRR8lAs9mZXETNPO9hYg3AkjUJBKi1NQ4aaIQZVGrig7c/3NUV1jANl8rFTeMg==} @@ -7044,11 +7037,6 @@ packages: tippy.js@6.3.7: resolution: {integrity: sha512-E1d3oP2emgJ9dRQZdf3Kkn0qJgI6ZLpyS5z6ZkY1DF3kaQaBsGZsndEpHwx+eC+tYM41HaSNvNtLx8tU57FzTQ==} - tiptap-markdown@0.8.10: - resolution: {integrity: sha512-iDVkR2BjAqkTDtFX0h94yVvE2AihCXlF0Q7RIXSJPRSR5I0PA1TMuAg6FHFpmqTn4tPxJ0by0CK7PUMlnFLGEQ==} - peerDependencies: - '@tiptap/core': ^2.0.3 - tmp@0.2.3: resolution: {integrity: sha512-nZD7m9iCPC5g0pYmcaxogYKggSfLsdxl8of3Q/oIbqCqLLIO9IAF0GWjX1z9NZRHPiXv8Wex4yDCaZsgEw0Y8w==} engines: {node: '>=14.14'} @@ -9471,8 +9459,6 @@ snapshots: '@types/json5@0.0.29': {} - '@types/linkify-it@3.0.5': {} - '@types/linkify-it@5.0.0': {} '@types/localforage@0.0.34': @@ -9481,18 +9467,11 @@ snapshots: '@types/lodash@4.17.20': {} - '@types/markdown-it@13.0.9': - dependencies: - '@types/linkify-it': 3.0.5 - '@types/mdurl': 1.0.5 - '@types/markdown-it@14.1.2': dependencies: '@types/linkify-it': 5.0.0 '@types/mdurl': 2.0.0 - '@types/mdurl@1.0.5': {} - '@types/mdurl@2.0.0': {} '@types/mime@1.3.5': {} @@ -13598,8 +13577,6 @@ snapshots: map-values@1.0.1: {} - markdown-it-task-lists@2.1.1: {} - markdown-it@14.1.0: dependencies: argparse: 2.0.1 @@ -13609,6 +13586,8 @@ snapshots: punycode.js: 2.3.1 uc.micro: 2.1.0 + marked@16.1.1: {} + marks-pane@1.0.9: {} material-icons@0.3.1: {} @@ -15773,14 +15752,6 @@ snapshots: dependencies: '@popperjs/core': 2.11.8 - tiptap-markdown@0.8.10(@tiptap/core@2.23.1(@tiptap/pm@2.23.1)): - dependencies: - '@tiptap/core': 2.23.1(@tiptap/pm@2.23.1) - '@types/markdown-it': 13.0.9 - markdown-it: 14.1.0 - markdown-it-task-lists: 2.1.1 - prosemirror-markdown: 1.13.2 - tmp@0.2.3: {} tmpl@1.0.5: {} From 2bae21dcdf796b1628d00d70f972b85423a07cc7 Mon Sep 17 00:00:00 2001 From: habibayman Date: Sun, 20 Jul 2025 05:15:29 +0300 Subject: [PATCH 023/472] config: add marked to jest transformIgnorePatterns --- jest_config/jest.conf.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jest_config/jest.conf.js b/jest_config/jest.conf.js index 20987fa663..b515327281 100644 --- a/jest_config/jest.conf.js +++ b/jest_config/jest.conf.js @@ -30,7 +30,7 @@ module.exports = { '.*\\.(vue)$': '/node_modules/vue-jest', }, transformIgnorePatterns: [ - '/node_modules/.pnpm/(?!(vuetify|epubjs|kolibri-design-system|kolibri-constants|axios|lowlight|@tiptap|tiptap|prosemirror-.*|unified|unist-.*|hast-.*|bail|trough|vfile.*|remark-.*|rehype-.*|mdast-.*|devlop))', + '/node_modules/.pnpm/(?!(vuetify|epubjs|kolibri-design-system|kolibri-constants|axios|lowlight|@tiptap|tiptap|prosemirror-.*|unified|unist-.*|hast-.*|bail|trough|vfile.*|remark-.*|rehype-.*|mdast-.*|devlop|marked))', ], snapshotSerializers: ['/node_modules/jest-serializer-vue'], setupFilesAfterEnv: ['/jest_config/setup.js'], From 807c8401be53792cdf7452eac0303685beb54851 Mon Sep 17 00:00:00 2001 From: habibayman Date: Mon, 21 Jul 2025 09:29:18 +0300 Subject: [PATCH 024/472] fix(texteditor)[markdown]: handle legacy images --- .../TipTapEditor/extensions/Image.js | 1 + .../TipTapEditor/utils/markdown.js | 43 +++++++++++++------ 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/extensions/Image.js b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/extensions/Image.js index 6b75c2eae1..140e1cd3d0 100644 --- a/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/extensions/Image.js +++ b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/extensions/Image.js @@ -14,6 +14,7 @@ export const Image = Node.create({ addAttributes() { return { src: { default: null }, + permanentSrc: { default: null }, alt: { default: null }, width: { default: null }, height: { default: null }, diff --git a/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/utils/markdown.js b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/utils/markdown.js index 38227c95d1..279dd451db 100644 --- a/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/utils/markdown.js +++ b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/utils/markdown.js @@ -1,9 +1,10 @@ // Contains utilities for handling Markdown content bidirectional conversion in TipTap editor. // eslint-disable-next-line import/namespace import { marked } from 'marked'; +import { storageUrl } from '../../../../vuex/file/utils'; // --- Image Translation --- -export const IMAGE_PLACEHOLDER = 'placeholder'; +export const IMAGE_PLACEHOLDER = '${☣ CONTENTSTORAGE}'; export const IMAGE_REGEX = /!\[([^\]]*)\]\(([^/]+\/[^\s=)]+)(?:\s*=\s*([0-9.]+)x([0-9.]+))?\)/g; export const imageMdToParams = markdown => { @@ -13,20 +14,31 @@ export const imageMdToParams = markdown => { if (!match) return null; const [, alt, fullPath, width, height] = match; - const pathParts = fullPath.split('/'); - // Ensure it matches the "placeholder/checksum.ext" structure - if (pathParts.length < 2 || pathParts[0] !== IMAGE_PLACEHOLDER) { - return null; - } + // Extract just the filename from the full path + const checksumWithExt = fullPath.split('/').pop(); - const src = pathParts.slice(1).join('/'); // The "checksum.ext" part + // Now, split the filename into its parts + const parts = checksumWithExt.split('.'); + const extension = parts.pop(); + const checksum = parts.join('.'); - return { src, alt: alt || '', width: width || null, height: height || null }; + // Return the data with the correct property names that the rest of the system expects. + return { checksum, extension, alt: alt || '', width, height }; }; -export const paramsToImageMd = ({ src, alt, width, height }) => { - const fileName = src.split('/').pop(); +export const paramsToImageMd = ({ src, alt, width, height, permanentSrc }) => { + const sourceToSave = permanentSrc || src; + + // As a safety net, if the source is still a data/blob URL, we should not + // try to create a placeholder format. This should not happen with our new logic, + // but it makes the function more robust. + if (sourceToSave.startsWith('data:') || sourceToSave.startsWith('blob:')) { + // TODO: This is not a good permanent format, temp fix + return `![${alt || ''}](${sourceToSave})`; + } + + const fileName = sourceToSave.split('/').pop(); if (width && height) { return `![${alt || ''}](${IMAGE_PLACEHOLDER}/${fileName} =${width}x${height})`; } @@ -58,11 +70,18 @@ export function preprocessMarkdown(markdown) { let processedMarkdown = markdown; - // First handle your custom syntax (images and math) as before processedMarkdown = processedMarkdown.replace(IMAGE_REGEX, match => { const params = imageMdToParams(match); if (!params) return match; - return `${params.alt}`; + + // 1. Convert the checksum into a real, displayable URL. + const displayUrl = storageUrl(params.checksum, params.extension); + + // 2. The permanentSrc is just the checksum + extension. + const permanentSrc = `${params.checksum}.${params.extension}`; + + // 3. Create an tag with the REAL display URL in `src`. + return `${params.alt}`; }); processedMarkdown = processedMarkdown.replace(MATH_REGEX, match => { From 9cddb63f73882d39f60bdc833480a6d5d172a162 Mon Sep 17 00:00:00 2001 From: Jakub Komarek Date: Mon, 21 Jul 2025 15:50:55 +0200 Subject: [PATCH 025/472] Address PR feedback --- ...communitylibrarysubmission_admin_fields.py | 1 + .../test_community_library_submission.py | 64 +++++++------ contentcuration/contentcuration/urls.py | 10 +- .../viewsets/community_library_submission.py | 96 +++++++++---------- 4 files changed, 84 insertions(+), 87 deletions(-) diff --git a/contentcuration/contentcuration/migrations/0156_communitylibrarysubmission_admin_fields.py b/contentcuration/contentcuration/migrations/0156_communitylibrarysubmission_admin_fields.py index e88594c47a..e7af358f71 100644 --- a/contentcuration/contentcuration/migrations/0156_communitylibrarysubmission_admin_fields.py +++ b/contentcuration/contentcuration/migrations/0156_communitylibrarysubmission_admin_fields.py @@ -34,6 +34,7 @@ class Migration(migrations.Migration): model_name="communitylibrarysubmission", name="resolution_reason", field=models.CharField( + blank=True, choices=[ ("INVALID_LICENSING", "Invalid Licensing"), ("TECHNICAL_QUALITY_ASSURANCE", "Technical Quality Assurance"), diff --git a/contentcuration/contentcuration/tests/viewsets/test_community_library_submission.py b/contentcuration/contentcuration/tests/viewsets/test_community_library_submission.py index cf21d9534a..4d2808658f 100644 --- a/contentcuration/contentcuration/tests/viewsets/test_community_library_submission.py +++ b/contentcuration/contentcuration/tests/viewsets/test_community_library_submission.py @@ -282,8 +282,10 @@ def test_get_single_submission__author_name(self): self.assertEqual(response.status_code, 200, response.content) result = response.data - self.assertEqual(result["author_first_name"], self.author_user.first_name) - self.assertEqual(result["author_last_name"], self.author_user.last_name) + self.assertEqual( + result["author_name"], + f"{self.author_user.first_name} {self.author_user.last_name}", + ) def test_update_submission__is_author(self): self.client.force_authenticate(user=self.author_user) @@ -477,7 +479,7 @@ def setUp(self): self.resolved_time = datetime.datetime(2023, 10, 1, tzinfo=pytz.utc) self.patcher = mock.patch( - "contentcuration.viewsets.community_library_submission.timezoned_datetime_now", + "contentcuration.viewsets.community_library_submission.timezone.now", return_value=self.resolved_time, ) self.mock_datetime = self.patcher.start() @@ -486,7 +488,7 @@ def tearDown(self): self.patcher.stop() super().tearDown() - def _manually_resolve_submission(self): + def _manually_reject_submission(self): self.submission.status = community_library_submission_constants.STATUS_REJECTED self.submission.resolved_by = self.admin_user self.submission.resolution_reason = ( @@ -506,10 +508,10 @@ def _refresh_submissions_from_db(self): def test_list_submissions__admin(self): self.client.force_authenticate(user=self.admin_user) - self._manually_resolve_submission() + self._manually_reject_submission() response = self.client.get( - reverse("community-library-submission-admin-list"), + reverse("admin-community-library-submission-list"), ) self.assertEqual(response.status_code, 200, response.content) @@ -525,28 +527,30 @@ def test_list_submissions__admin(self): result = rejected_results[0] self.assertEqual(result["resolved_by_id"], self.admin_user.id) - self.assertEqual(result["resolved_by_first_name"], self.admin_user.first_name) - self.assertEqual(result["resolved_by_last_name"], self.admin_user.last_name) + self.assertEqual( + result["resolved_by_name"], + f"{self.admin_user.first_name} {self.admin_user.last_name}", + ) self.assertEqual(result["internal_notes"], self.internal_notes) def test_list_submissions__editor(self): self.client.force_authenticate(user=self.editor_user) - self._manually_resolve_submission() + self._manually_reject_submission() response = self.client.get( - reverse("community-library-submission-admin-list"), + reverse("admin-community-library-submission-list"), ) self.assertEqual(response.status_code, 403, response.content) def test_submission_detail__admin(self): self.client.force_authenticate(user=self.admin_user) - self._manually_resolve_submission() + self._manually_reject_submission() response = self.client.get( reverse( - "community-library-submission-admin-detail", + "admin-community-library-submission-detail", args=[self.submission.id], ), ) @@ -555,18 +559,20 @@ def test_submission_detail__admin(self): result = response.data self.assertEqual(result["id"], self.submission.id) self.assertEqual(result["resolved_by_id"], self.admin_user.id) - self.assertEqual(result["resolved_by_first_name"], self.admin_user.first_name) - self.assertEqual(result["resolved_by_last_name"], self.admin_user.last_name) + self.assertEqual( + result["resolved_by_name"], + f"{self.admin_user.first_name} {self.admin_user.last_name}", + ) self.assertEqual(result["internal_notes"], self.internal_notes) def test_submission_detail__editor(self): self.client.force_authenticate(user=self.editor_user) - self._manually_resolve_submission() + self._manually_reject_submission() response = self.client.get( reverse( - "community-library-submission-admin-detail", + "admin-community-library-submission-detail", args=[self.submission.id], ), ) @@ -577,7 +583,7 @@ def test_update_submission(self): response = self.client.put( reverse( - "community-library-submission-admin-detail", + "admin-community-library-submission-detail", args=[self.submission.id], ), {}, @@ -590,7 +596,7 @@ def test_partial_update_submission(self): response = self.client.patch( reverse( - "community-library-submission-admin-detail", + "admin-community-library-submission-detail", args=[self.submission.id], ), {}, @@ -603,7 +609,7 @@ def test_destroy_submission(self): response = self.client.delete( reverse( - "community-library-submission-admin-detail", + "admin-community-library-submission-detail", args=[self.submission.id], ), format="json", @@ -614,7 +620,7 @@ def test_resolve_submission__editor(self): self.client.force_authenticate(user=self.editor_user) response = self.client.post( reverse( - "community-library-submission-admin-resolve", + "admin-community-library-submission-resolve", args=[self.submission.id], ), self.resolve_approve_metadata, @@ -626,7 +632,7 @@ def test_resolve_submission__accept_correct(self): self.client.force_authenticate(user=self.admin_user) response = self.client.post( reverse( - "community-library-submission-admin-resolve", + "admin-community-library-submission-resolve", args=[self.submission.id], ), self.resolve_approve_metadata, @@ -650,7 +656,7 @@ def test_resolve_submission__reject_correct(self): self.client.force_authenticate(user=self.admin_user) response = self.client.post( reverse( - "community-library-submission-admin-resolve", + "admin-community-library-submission-resolve", args=[self.submission.id], ), self.resolve_reject_metadata, @@ -680,7 +686,7 @@ def test_resolve_submission__reject_missing_resolution_reason(self): del metadata["resolution_reason"] response = self.client.post( reverse( - "community-library-submission-admin-resolve", + "admin-community-library-submission-resolve", args=[self.submission.id], ), metadata, @@ -694,7 +700,7 @@ def test_resolve_submission__reject_missing_feedback_notes(self): del metadata["feedback_notes"] response = self.client.post( reverse( - "community-library-submission-admin-resolve", + "admin-community-library-submission-resolve", args=[self.submission.id], ), metadata, @@ -708,7 +714,7 @@ def test_resolve_submission__invalid_status(self): metadata["status"] = (community_library_submission_constants.STATUS_PENDING,) response = self.client.post( reverse( - "community-library-submission-admin-resolve", + "admin-community-library-submission-resolve", args=[self.submission.id], ), metadata, @@ -723,7 +729,7 @@ def test_resolve_submission__not_pending(self): response = self.client.post( reverse( - "community-library-submission-admin-resolve", + "admin-community-library-submission-resolve", args=[self.submission.id], ), self.resolve_approve_metadata, @@ -738,7 +744,7 @@ def test_resolve_submission__overrite_categories(self): response = self.client.post( reverse( - "community-library-submission-admin-resolve", + "admin-community-library-submission-resolve", args=[self.submission.id], ), self.resolve_approve_metadata, @@ -756,7 +762,7 @@ def test_resolve_submission__accept_mark_superseded(self): response = self.client.post( reverse( - "community-library-submission-admin-resolve", + "admin-community-library-submission-resolve", args=[self.submission.id], ), self.resolve_approve_metadata, @@ -784,7 +790,7 @@ def test_resolve_submission__reject_do_not_mark_superseded(self): response = self.client.post( reverse( - "community-library-submission-admin-resolve", + "admin-community-library-submission-resolve", args=[self.submission.id], ), self.resolve_reject_metadata, diff --git a/contentcuration/contentcuration/urls.py b/contentcuration/contentcuration/urls.py index 7216eb972d..7ccb5b53a5 100644 --- a/contentcuration/contentcuration/urls.py +++ b/contentcuration/contentcuration/urls.py @@ -39,7 +39,7 @@ from contentcuration.viewsets.channelset import ChannelSetViewSet from contentcuration.viewsets.clipboard import ClipboardViewSet from contentcuration.viewsets.community_library_submission import ( - CommunityLibrarySubmissionAdminViewSet, + AdminCommunityLibrarySubmissionViewSet, ) from contentcuration.viewsets.community_library_submission import ( CommunityLibrarySubmissionViewSet, @@ -87,14 +87,14 @@ def get_redirect_url(self, *args, **kwargs): basename="recommendations-interaction", ) router.register( - r"communitylibrary_submission", + r"communitylibrary-submission", CommunityLibrarySubmissionViewSet, basename="community-library-submission", ) router.register( - r"communitylibrary_submission_admin", - CommunityLibrarySubmissionAdminViewSet, - basename="community-library-submission-admin", + r"admin_communitylibrary_submission", + AdminCommunityLibrarySubmissionViewSet, + basename="admin-community-library-submission", ) urlpatterns = [ diff --git a/contentcuration/contentcuration/viewsets/community_library_submission.py b/contentcuration/contentcuration/viewsets/community_library_submission.py index 7610ae323e..6d45e0c4e4 100644 --- a/contentcuration/contentcuration/viewsets/community_library_submission.py +++ b/contentcuration/contentcuration/viewsets/community_library_submission.py @@ -1,8 +1,6 @@ -import datetime - +from django.utils import timezone from django_filters.rest_framework import DjangoFilterBackend from rest_framework.decorators import action -from rest_framework.exceptions import MethodNotAllowed from rest_framework.permissions import IsAuthenticated from rest_framework.relations import PrimaryKeyRelatedField from rest_framework.response import Response @@ -91,16 +89,6 @@ def update(self, instance, validated_data): return super().update(instance, validated_data) -def timezoned_datetime_now(): - """ - Simple wrapper around datetime.datetime.now. The point of using a wrapper - is that mocking datetime.datetime in tests runs into issues with - isinstance in the serializer, and it is a lot easier to just mock this - wrapper. - """ - return datetime.datetime.now(datetime.timezone.utc) - - class CommunityLibrarySubmissionResolveSerializer(CommunityLibrarySubmissionSerializer): class Meta(CommunityLibrarySubmissionSerializer.Meta): fields = CommunityLibrarySubmissionSerializer.Meta.fields + [ @@ -135,11 +123,11 @@ def update(self, instance, validated_data): or validated_data["status"] == community_library_submission_constants.STATUS_REJECTED ): - if "resolution_reason" not in validated_data: + if not validated_data.get("resolution_reason", "").strip(): raise ValidationError( "Resolution reason must be provided when rejecting a submission." ) - if "feedback_notes" not in validated_data: + if not validated_data.get("feedback_notes", "").strip(): raise ValidationError( "Feedback notes must be provided when rejecting a submission." ) @@ -153,12 +141,16 @@ class CommunityLibrarySubmissionPagination(ValuesViewsetCursorPagination): max_page_size = 100 -class CommunityLibrarySubmissionViewSet( - RESTCreateModelMixin, - RESTUpdateModelMixin, - RESTDestroyModelMixin, - ReadOnlyValuesViewset, -): +def get_author_name(item): + return "{} {}".format(item["author__first_name"], item["author__last_name"]) + + +class CommunityLibrarySubmissionViewSetMixin: + """ + Mixin with logic shared between the CommunityLibrarySubmissionViewSet and + AdminCommunityLibrarySubmissionViewSet. + """ + values = ( "id", "description", @@ -175,14 +167,8 @@ class CommunityLibrarySubmissionViewSet( "date_resolved", ) field_map = { - "author_first_name": "author__first_name", - "author_last_name": "author__last_name", + "author_name": get_author_name, } - filter_backends = [DjangoFilterBackend] - filterset_fields = ["channel"] - permission_classes = [IsAuthenticated] - pagination_class = CommunityLibrarySubmissionPagination - serializer_class = CommunityLibrarySubmissionSerializer queryset = CommunityLibrarySubmission.objects.all().order_by("-date_created") def consolidate(self, items, queryset): @@ -200,41 +186,45 @@ def consolidate(self, items, queryset): return items -class CommunityLibrarySubmissionAdminViewSet(CommunityLibrarySubmissionViewSet): +class CommunityLibrarySubmissionViewSet( + CommunityLibrarySubmissionViewSetMixin, + RESTCreateModelMixin, + RESTUpdateModelMixin, + RESTDestroyModelMixin, + ReadOnlyValuesViewset, +): + filter_backends = [DjangoFilterBackend] + filterset_fields = ["channel"] + permission_classes = [IsAuthenticated] + pagination_class = CommunityLibrarySubmissionPagination + serializer_class = CommunityLibrarySubmissionSerializer + + +def get_resolved_by_name(item): + return "{} {}".format( + item["resolved_by__first_name"], item["resolved_by__last_name"] + ) + + +class AdminCommunityLibrarySubmissionViewSet( + CommunityLibrarySubmissionViewSetMixin, + ReadOnlyValuesViewset, +): permission_classes = [IsAdminUser] - values = CommunityLibrarySubmissionViewSet.values + ( + values = CommunityLibrarySubmissionViewSetMixin.values + ( "resolved_by_id", "resolved_by__first_name", "resolved_by__last_name", "internal_notes", ) - field_map = CommunityLibrarySubmissionViewSet.field_map.copy() + field_map = CommunityLibrarySubmissionViewSetMixin.field_map.copy() field_map.update( { - "resolved_by_first_name": "resolved_by__first_name", - "resolved_by_last_name": "resolved_by__last_name", + "resolved_by_name": get_resolved_by_name, } ) - def create(self, request, *args, **kwargs): - raise MethodNotAllowed( - "Cannot create a community library submission with this viewset. " - "Use the standard CommunityLibrarySubmissionViewSet instead." - ) - - def update(self, instance, *args, **kwargs): - raise MethodNotAllowed( - "Cannot update a community library submission with this viewset. " - "Use the standard CommunityLibrarySubmissionViewSet instead." - ) - - def destroy(self, instance, *args, **kwargs): - raise MethodNotAllowed( - "Cannot delete a community library submission with this viewset. " - "Use the standard CommunityLibrarySubmissionViewSet instead." - ) - def _mark_previous_pending_submissions_as_superseded(self, submission): CommunityLibrarySubmission.objects.filter( status=community_library_submission_constants.STATUS_PENDING, @@ -252,7 +242,7 @@ def resolve(self, request, pk=None): serializer = self.get_serializer(instance, data=request.data, partial=True) serializer.is_valid(raise_exception=True) - date_resolved = timezoned_datetime_now() + date_resolved = timezone.now() submission = serializer.save( date_resolved=date_resolved, resolved_by=request.user, @@ -261,4 +251,4 @@ def resolve(self, request, pk=None): if submission.status == community_library_submission_constants.STATUS_APPROVED: self._mark_previous_pending_submissions_as_superseded(submission) - return Response(serializer.data) + return Response(self.serialize_object(pk=submission.id)) From 01f25a3d7d16529fb9a385e465d68848b47e00f9 Mon Sep 17 00:00:00 2001 From: habibayman Date: Mon, 21 Jul 2025 19:03:29 +0300 Subject: [PATCH 026/472] fix(texteditor)[markdown]: save img dimensions properly --- .../components/image/ImageNodeView.vue | 88 +++++++++++++++++-- .../TipTapEditor/utils/markdown.js | 17 +++- 2 files changed, 95 insertions(+), 10 deletions(-) diff --git a/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/components/image/ImageNodeView.vue b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/components/image/ImageNodeView.vue index 0e16eb11d4..b2e78c3387 100644 --- a/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/components/image/ImageNodeView.vue +++ b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/components/image/ImageNodeView.vue @@ -7,9 +7,11 @@ :style="{ width: styleWidth }" >
- import { defineComponent, ref, computed, onUnmounted, watch } from 'vue'; + import { defineComponent, ref, computed, onUnmounted, onMounted, watch } from 'vue'; import { NodeViewWrapper } from '@tiptap/vue-2'; export default defineComponent({ @@ -72,12 +74,15 @@ NodeViewWrapper, }, setup(props) { - const width = ref(props.node.attrs.width); + const width = ref(props.node.attrs.width || null); + const height = ref(props.node.attrs.height || null); + const imageRef = ref(null); + const naturalAspectRatio = ref(null); const minWidth = 50; const compactThreshold = 200; let debounceTimer = null; - // (to work with undo/redo) Watch for external changes to the node's width + // (to work with undo/redo) Watch for external changes to the node's width and height watch( () => props.node.attrs.width, newWidth => { @@ -85,6 +90,47 @@ }, ); + watch( + () => props.node.attrs.height, + newHeight => { + height.value = newHeight; + }, + ); + + // Watch for src changes to recalculate aspect ratio + watch( + () => props.node.attrs.src, + () => { + // Reset aspect ratio when src changes + naturalAspectRatio.value = null; + // Force image to reload and recalculate dimensions + if (imageRef.value) { + imageRef.value.onload = onImageLoad; + } + }, + ); + + const onImageLoad = () => { + if (imageRef.value && imageRef.value.naturalWidth && imageRef.value.naturalHeight) { + naturalAspectRatio.value = imageRef.value.naturalWidth / imageRef.value.naturalHeight; + + // If no dimensions are set, use natural dimensions + if (!width.value && !height.value) { + width.value = imageRef.value.naturalWidth; + height.value = imageRef.value.naturalHeight; + saveSize(); + } else if (width.value && !height.value) { + // If we have width but no height, calculate height + height.value = calculateProportionalHeight(width.value); + saveSize(); + } else if (!width.value && height.value) { + // If we have height but no width, calculate width + width.value = Math.round(height.value * naturalAspectRatio.value); + saveSize(); + } + } + }; + const isRtl = computed(() => { return props.editor.view.dom.closest('[dir="rtl"]') !== null; }); @@ -92,13 +138,25 @@ const styleWidth = computed(() => (width.value ? `${width.value}px` : 'auto')); const isCompact = computed(() => width.value < compactThreshold); - const saveWidth = () => { + const saveSize = () => { props.updateAttributes({ width: width.value, - height: null, + height: height.value, }); }; + const calculateProportionalHeight = newWidth => { + if (naturalAspectRatio.value) { + return Math.round(newWidth / naturalAspectRatio.value); + } + // Fallback: try to get aspect ratio directly from the image element + if (imageRef.value && imageRef.value.naturalWidth && imageRef.value.naturalHeight) { + const ratio = imageRef.value.naturalWidth / imageRef.value.naturalHeight; + return Math.round(newWidth / ratio); + } + return null; + }; + const onResizeStart = startEvent => { const startX = startEvent.clientX; const startWidth = width.value || startEvent.target.parentElement.offsetWidth; @@ -110,13 +168,15 @@ ? startWidth - deltaX // In RTL, moving right should decrease width : startWidth + deltaX; // In LTR, moving right should increase width - width.value = Math.max(minWidth, newWidth); + const clampedWidth = Math.max(minWidth, newWidth); + width.value = clampedWidth; + height.value = calculateProportionalHeight(clampedWidth); }; const onMouseUp = () => { document.removeEventListener('mousemove', onMouseMove); document.removeEventListener('mouseup', onMouseUp); - saveWidth(); + saveSize(); }; document.addEventListener('mousemove', onMouseMove); @@ -145,10 +205,12 @@ return; } - width.value = Math.max(minWidth, newWidth); + const clampedWidth = Math.max(minWidth, newWidth); + width.value = clampedWidth; + height.value = calculateProportionalHeight(clampedWidth); clearTimeout(debounceTimer); - debounceTimer = setTimeout(saveWidth, 500); + debounceTimer = setTimeout(saveSize, 500); }; const removeImage = () => { @@ -166,18 +228,26 @@ }); }; + onMounted(() => { + if (imageRef.value && imageRef.value.complete) { + onImageLoad(); + } + }); + onUnmounted(() => { clearTimeout(debounceTimer); }); return { width, + imageRef, styleWidth, onResizeStart, removeImage, editImage, isCompact, onResizeKeyDown, + onImageLoad, }; }, props: { diff --git a/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/utils/markdown.js b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/utils/markdown.js index 279dd451db..0ac3204ade 100644 --- a/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/utils/markdown.js +++ b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/utils/markdown.js @@ -6,6 +6,8 @@ import { storageUrl } from '../../../../vuex/file/utils'; // --- Image Translation --- export const IMAGE_PLACEHOLDER = '${☣ CONTENTSTORAGE}'; export const IMAGE_REGEX = /!\[([^\]]*)\]\(([^/]+\/[^\s=)]+)(?:\s*=\s*([0-9.]+)x([0-9.]+))?\)/g; +export const DATA_URL_IMAGE_REGEX = + /!\[([^\]]*)\]\(((?:data:|blob:).+?)(?:\s*=\s*([0-9.]+)x([0-9.]+))?\)/g; export const imageMdToParams = markdown => { // Reset regex state before executing to ensure it works on all matches @@ -34,7 +36,9 @@ export const paramsToImageMd = ({ src, alt, width, height, permanentSrc }) => { // try to create a placeholder format. This should not happen with our new logic, // but it makes the function more robust. if (sourceToSave.startsWith('data:') || sourceToSave.startsWith('blob:')) { - // TODO: This is not a good permanent format, temp fix + if (width && height) { + return `![${alt || ''}](${sourceToSave} =${width}x${height})`; + } return `![${alt || ''}](${sourceToSave})`; } @@ -70,6 +74,17 @@ export function preprocessMarkdown(markdown) { let processedMarkdown = markdown; + // First, handle data URLs and blob URLs for images. + processedMarkdown = processedMarkdown.replace( + DATA_URL_IMAGE_REGEX, + (match, alt, src, width, height) => { + const widthAttr = width ? ` width="${width}"` : ''; + const heightAttr = height ? ` height="${height}"` : ''; + return `${alt || ''}`; + }, + ); + + // Then, handle our standard content-storage images. processedMarkdown = processedMarkdown.replace(IMAGE_REGEX, match => { const params = imageMdToParams(match); if (!params) return match; From ff53d60c24dc3ff15f029d8d17fbb0a153ac2227 Mon Sep 17 00:00:00 2001 From: Jakub Komarek Date: Mon, 21 Jul 2025 19:02:07 +0200 Subject: [PATCH 027/472] Use the same case for both community library routes --- contentcuration/contentcuration/urls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contentcuration/contentcuration/urls.py b/contentcuration/contentcuration/urls.py index 7ccb5b53a5..f65d9fa35c 100644 --- a/contentcuration/contentcuration/urls.py +++ b/contentcuration/contentcuration/urls.py @@ -87,7 +87,7 @@ def get_redirect_url(self, *args, **kwargs): basename="recommendations-interaction", ) router.register( - r"communitylibrary-submission", + r"communitylibrary_submission", CommunityLibrarySubmissionViewSet, basename="community-library-submission", ) From 73ad4cbc6d951789f7d1a5a4beb1c31da39bbb68 Mon Sep 17 00:00:00 2001 From: Jakub Komarek Date: Mon, 21 Jul 2025 19:05:36 +0200 Subject: [PATCH 028/472] Do not pass pk explicitly to serialize_object --- .../contentcuration/viewsets/community_library_submission.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contentcuration/contentcuration/viewsets/community_library_submission.py b/contentcuration/contentcuration/viewsets/community_library_submission.py index 6d45e0c4e4..eb5e2334c3 100644 --- a/contentcuration/contentcuration/viewsets/community_library_submission.py +++ b/contentcuration/contentcuration/viewsets/community_library_submission.py @@ -251,4 +251,4 @@ def resolve(self, request, pk=None): if submission.status == community_library_submission_constants.STATUS_APPROVED: self._mark_previous_pending_submissions_as_superseded(submission) - return Response(self.serialize_object(pk=submission.id)) + return Response(self.serialize_object()) From 72bbef69118ea84deddee00144574da272cbb5a7 Mon Sep 17 00:00:00 2001 From: Jakub Komarek Date: Mon, 21 Jul 2025 19:07:41 +0200 Subject: [PATCH 029/472] Move community library filtering and pagination to mixin --- .../viewsets/community_library_submission.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/contentcuration/contentcuration/viewsets/community_library_submission.py b/contentcuration/contentcuration/viewsets/community_library_submission.py index eb5e2334c3..163ab50e77 100644 --- a/contentcuration/contentcuration/viewsets/community_library_submission.py +++ b/contentcuration/contentcuration/viewsets/community_library_submission.py @@ -170,6 +170,9 @@ class CommunityLibrarySubmissionViewSetMixin: "author_name": get_author_name, } queryset = CommunityLibrarySubmission.objects.all().order_by("-date_created") + filter_backends = [DjangoFilterBackend] + filterset_fields = ["channel"] + pagination_class = CommunityLibrarySubmissionPagination def consolidate(self, items, queryset): countries = {} @@ -193,10 +196,7 @@ class CommunityLibrarySubmissionViewSet( RESTDestroyModelMixin, ReadOnlyValuesViewset, ): - filter_backends = [DjangoFilterBackend] - filterset_fields = ["channel"] permission_classes = [IsAuthenticated] - pagination_class = CommunityLibrarySubmissionPagination serializer_class = CommunityLibrarySubmissionSerializer From 7a3c46e6c21d86563fc4ebe46ab9d00600eccfc6 Mon Sep 17 00:00:00 2001 From: Jakub Komarek Date: Thu, 24 Jul 2025 22:39:27 +0200 Subject: [PATCH 030/472] Remove leftover migration --- ...nitylibrarysubmission_resolution_reason.py | 29 ------------------- 1 file changed, 29 deletions(-) delete mode 100644 contentcuration/contentcuration/migrations/0157_alter_communitylibrarysubmission_resolution_reason.py diff --git a/contentcuration/contentcuration/migrations/0157_alter_communitylibrarysubmission_resolution_reason.py b/contentcuration/contentcuration/migrations/0157_alter_communitylibrarysubmission_resolution_reason.py deleted file mode 100644 index 09a4f7d190..0000000000 --- a/contentcuration/contentcuration/migrations/0157_alter_communitylibrarysubmission_resolution_reason.py +++ /dev/null @@ -1,29 +0,0 @@ -# Generated by Django 3.2.24 on 2025-07-15 20:11 -from django.db import migrations -from django.db import models - - -class Migration(migrations.Migration): - - dependencies = [ - ("contentcuration", "0156_communitylibrarysubmission_admin_fields"), - ] - - operations = [ - migrations.AlterField( - model_name="communitylibrarysubmission", - name="resolution_reason", - field=models.CharField( - blank=True, - choices=[ - ("INVALID_LICENSING", "Invalid Licensing"), - ("TECHNICAL_QUALITY_ASSURANCE", "Technical Quality Assurance"), - ("INVALID_METADATA", "Invalid Metadata"), - ("PORTABILITY_ISSUES", "Portability Issues"), - ("OTHER", "Other"), - ], - max_length=50, - null=True, - ), - ), - ] From 84bda22cd5e649ba4eebb052cd020b13ee6be677 Mon Sep 17 00:00:00 2001 From: Alex Velez Date: Mon, 28 Jul 2025 10:56:18 -0500 Subject: [PATCH 031/472] Copy side panel modal to Studio --- .../frontend/shared/strings/commonStrings.js | 13 + .../shared/views/SidePanelModal/Backdrop.vue | 75 +++++ .../shared/views/SidePanelModal/index.vue | 274 ++++++++++++++++++ 3 files changed, 362 insertions(+) create mode 100644 contentcuration/contentcuration/frontend/shared/strings/commonStrings.js create mode 100644 contentcuration/contentcuration/frontend/shared/views/SidePanelModal/Backdrop.vue create mode 100644 contentcuration/contentcuration/frontend/shared/views/SidePanelModal/index.vue diff --git a/contentcuration/contentcuration/frontend/shared/strings/commonStrings.js b/contentcuration/contentcuration/frontend/shared/strings/commonStrings.js new file mode 100644 index 0000000000..7b3137c2e2 --- /dev/null +++ b/contentcuration/contentcuration/frontend/shared/strings/commonStrings.js @@ -0,0 +1,13 @@ +import { createTranslator } from 'shared/i18n'; + +export const commonStrings = createTranslator('CommonStrings', { + backAction: { + message: 'Back', + context: + 'Indicates going back to a previous step in multi-step workflows. It can be used as a label of the back button that is displayed next to the continue button.', + }, + closeAction: { + message: 'Close', + context: 'A label for an action that closes a dialog or window', + }, +}); diff --git a/contentcuration/contentcuration/frontend/shared/views/SidePanelModal/Backdrop.vue b/contentcuration/contentcuration/frontend/shared/views/SidePanelModal/Backdrop.vue new file mode 100644 index 0000000000..af8d2b99e1 --- /dev/null +++ b/contentcuration/contentcuration/frontend/shared/views/SidePanelModal/Backdrop.vue @@ -0,0 +1,75 @@ + + + + + + + diff --git a/contentcuration/contentcuration/frontend/shared/views/SidePanelModal/index.vue b/contentcuration/contentcuration/frontend/shared/views/SidePanelModal/index.vue new file mode 100644 index 0000000000..30b695e5d0 --- /dev/null +++ b/contentcuration/contentcuration/frontend/shared/views/SidePanelModal/index.vue @@ -0,0 +1,274 @@ + + + + + + + From ba03ab90dffb312edcfc1e24ee37f8704b5386d3 Mon Sep 17 00:00:00 2001 From: Alex Velez Date: Mon, 28 Jul 2025 10:56:55 -0500 Subject: [PATCH 032/472] Add community channels strings file --- .../frontend/shared/strings/communityChannelsStrings.js | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 contentcuration/contentcuration/frontend/shared/strings/communityChannelsStrings.js diff --git a/contentcuration/contentcuration/frontend/shared/strings/communityChannelsStrings.js b/contentcuration/contentcuration/frontend/shared/strings/communityChannelsStrings.js new file mode 100644 index 0000000000..489af9eb43 --- /dev/null +++ b/contentcuration/contentcuration/frontend/shared/strings/communityChannelsStrings.js @@ -0,0 +1,3 @@ +import { createTranslator } from 'shared/i18n'; + +export const communityChannelsStrings = createTranslator('CommunityChannelsStrings', {}); From 0043f6d0dd29e27d2e9f66b3a3a9735a07cd87ee Mon Sep 17 00:00:00 2001 From: Jakub Komarek Date: Wed, 30 Jul 2025 13:59:29 +0200 Subject: [PATCH 033/472] Add to community library on submission resolve --- .../tests/viewsets/test_channel.py | 87 ++++++++++ .../test_community_library_submission.py | 31 +++- .../contentcuration/viewsets/channel.py | 46 ++++++ .../viewsets/community_library_submission.py | 25 +++ .../contentcuration/viewsets/sync/base.py | 2 + .../viewsets/sync/constants.py | 8 + .../contentcuration/viewsets/sync/endpoint.py | 7 +- .../contentcuration/viewsets/sync/utils.py | 20 +++ .../export_channels_to_kolibri_public.py | 38 +---- .../migrations/0007_new_channel_metadata.py | 26 +++ contentcuration/kolibri_public/models.py | 3 + .../test_export_channels_to_kolibri_public.py | 152 +++++++++++++++++ .../kolibri_public/tests/test_mapper.py | 154 +++++++++++++++--- .../kolibri_public/utils/annotation.py | 42 ++++- .../utils/export_channel_to_kolibri_public.py | 60 +++++++ .../kolibri_public/utils/mapper.py | 26 ++- 16 files changed, 663 insertions(+), 64 deletions(-) create mode 100644 contentcuration/kolibri_public/migrations/0007_new_channel_metadata.py create mode 100644 contentcuration/kolibri_public/tests/test_export_channels_to_kolibri_public.py create mode 100644 contentcuration/kolibri_public/utils/export_channel_to_kolibri_public.py diff --git a/contentcuration/contentcuration/tests/viewsets/test_channel.py b/contentcuration/contentcuration/tests/viewsets/test_channel.py index 067e442ede..9ea043966c 100644 --- a/contentcuration/contentcuration/tests/viewsets/test_channel.py +++ b/contentcuration/contentcuration/tests/viewsets/test_channel.py @@ -11,7 +11,10 @@ from contentcuration import models from contentcuration import models as cc from contentcuration.constants import channel_history +from contentcuration.models import Change from contentcuration.models import ContentNode +from contentcuration.models import Country +from contentcuration.tasks import apply_channel_changes_task from contentcuration.tests import testdata from contentcuration.tests.base import StudioAPITestCase from contentcuration.tests.viewsets.base import generate_create_event @@ -24,6 +27,9 @@ from contentcuration.tests.viewsets.base import SyncTestMixin from contentcuration.viewsets.channel import _unpublished_changes_query from contentcuration.viewsets.sync.constants import CHANNEL +from contentcuration.viewsets.sync.utils import ( + generate_added_to_community_library_event, +) class SyncTestCase(SyncTestMixin, StudioAPITestCase): @@ -517,6 +523,87 @@ def test_publish_next_with_incomplete_staging_tree(self): modified_channel = models.Channel.objects.get(id=channel.id) self.assertEqual(modified_channel.staging_tree.published, False) + def test_sync_added_to_community_library_change(self): + # Syncing the change from the frontend should be disallowed + self.client.force_authenticate(self.admin_user) + + channel = testdata.channel() + channel.version = 1 + channel.public = True + channel.save() + + added_to_community_library_change = generate_added_to_community_library_event( + key=channel.id, + channel_version=1, + ) + response = self.sync_changes([added_to_community_library_change]) + + self.assertEqual(response.status_code, 200, response.content) + self.assertEqual(len(response.json()["allowed"]), 0, response.content) + self.assertEqual(len(response.json()["disallowed"]), 1, response.content) + + @mock.patch("contentcuration.viewsets.channel.export_channel_to_kolibri_public") + def test_process_added_to_community_library_change(self, mock_export_func): + # Creating the change on the backend should be supported + self.client.force_authenticate(self.admin_user) + + channel = testdata.channel() + channel.version = 1 + channel.public = True + channel.save() + + categories = { + "category1": True, + "category2": True, + } + + country1 = Country.objects.create(code="C1", name="Country 1") + country2 = Country.objects.create(code="C2", name="Country 2") + countries = [country1, country2] + country_codes = [country.code for country in countries] + + added_to_community_library_change = generate_added_to_community_library_event( + key=channel.id, + channel_version=1, + categories=categories, + country_codes=country_codes, + ) + Change.create_change( + added_to_community_library_change, created_by_id=self.admin_user.id + ) + + # This task will run immediately thanks to SyncTestMixin + apply_channel_changes_task.fetch_or_enqueue( + user=self.admin_user, + channel_id=channel.id, + ) + + # We cannot easily use the assert_called_once_with method here + # because we are not checking countries for strict equality, + # so we need to check the call arguments manually + mock_export_func.assert_called_once() + + (call_args, call_kwargs) = mock_export_func.call_args + self.assertEqual(len(call_args), 0) + self.assertCountEqual( + call_kwargs.keys(), + [ + "channel_id", + "channel_version", + "categories", + "countries", + "public", + ], + ) + self.assertEqual(call_kwargs["channel_id"], channel.id) + self.assertEqual(call_kwargs["channel_version"], 1) + self.assertEqual(call_kwargs["categories"], categories) + + # The countries argument used when creating the mapper is in fact + # not a list, but a QuerySet, but it contains the same elements + self.assertCountEqual(call_kwargs["countries"], countries) + self.assertEqual(call_kwargs["public"], False) + class CRUDTestCase(StudioAPITestCase): @property diff --git a/contentcuration/contentcuration/tests/viewsets/test_community_library_submission.py b/contentcuration/contentcuration/tests/viewsets/test_community_library_submission.py index 4d2808658f..f0fff43647 100644 --- a/contentcuration/contentcuration/tests/viewsets/test_community_library_submission.py +++ b/contentcuration/contentcuration/tests/viewsets/test_community_library_submission.py @@ -8,9 +8,11 @@ from contentcuration.constants import ( community_library_submission as community_library_submission_constants, ) +from contentcuration.models import Change from contentcuration.models import CommunityLibrarySubmission from contentcuration.tests import testdata from contentcuration.tests.base import StudioAPITestCase +from contentcuration.viewsets.sync.constants import ADDED_TO_COMMUNITY_LIBRARY def reverse_with_query( @@ -628,7 +630,10 @@ def test_resolve_submission__editor(self): ) self.assertEqual(response.status_code, 403, response.content) - def test_resolve_submission__accept_correct(self): + @mock.patch( + "contentcuration.viewsets.community_library_submission.apply_channel_changes_task" + ) + def test_resolve_submission__accept_correct(self, apply_task_mock): self.client.force_authenticate(user=self.admin_user) response = self.client.post( reverse( @@ -652,7 +657,21 @@ def test_resolve_submission__accept_correct(self): self.assertEqual(resolved_submission.resolved_by, self.admin_user) self.assertEqual(resolved_submission.date_resolved, self.resolved_time) - def test_resolve_submission__reject_correct(self): + self.assertTrue( + Change.objects.filter( + channel=self.submission.channel, + change_type=ADDED_TO_COMMUNITY_LIBRARY, + ).exists() + ) + apply_task_mock.fetch_or_enqueue.assert_called_once_with( + self.admin_user, + channel_id=self.submission.channel.id, + ) + + @mock.patch( + "contentcuration.viewsets.community_library_submission.apply_channel_changes_task" + ) + def test_resolve_submission__reject_correct(self, apply_task_mock): self.client.force_authenticate(user=self.admin_user) response = self.client.post( reverse( @@ -680,6 +699,14 @@ def test_resolve_submission__reject_correct(self): self.assertEqual(resolved_submission.resolved_by, self.admin_user) self.assertEqual(resolved_submission.date_resolved, self.resolved_time) + self.assertFalse( + Change.objects.filter( + channel=self.submission.channel, + change_type=ADDED_TO_COMMUNITY_LIBRARY, + ).exists() + ) + apply_task_mock.fetch_or_enqueue.assert_not_called() + def test_resolve_submission__reject_missing_resolution_reason(self): self.client.force_authenticate(user=self.admin_user) metadata = self.resolve_reject_metadata.copy() diff --git a/contentcuration/contentcuration/viewsets/channel.py b/contentcuration/contentcuration/viewsets/channel.py index 1fd6625030..5f8a33ade9 100644 --- a/contentcuration/contentcuration/viewsets/channel.py +++ b/contentcuration/contentcuration/viewsets/channel.py @@ -21,6 +21,9 @@ from django_cte import With from django_filters.rest_framework import BooleanFilter from django_filters.rest_framework import CharFilter +from kolibri_public.utils.export_channel_to_kolibri_public import ( + export_channel_to_kolibri_public, +) from le_utils.constants import content_kinds from le_utils.constants import roles from rest_framework import serializers @@ -43,6 +46,7 @@ from contentcuration.models import Change from contentcuration.models import Channel from contentcuration.models import ContentNode +from contentcuration.models import Country from contentcuration.models import File from contentcuration.models import generate_storage_url from contentcuration.models import SecretToken @@ -768,6 +772,48 @@ def deploy(self, user, pk): created_by_id=user.id, ) + def add_to_community_library_from_changes(self, changes): + errors = [] + for change in changes: + try: + self.add_to_community_library( + channel_id=change["channel_id"], + channel_version=change["channel_version"], + categories=change["categories"], + country_codes=change["country_codes"], + ) + except Exception as e: + log_sync_exception(e, user=self.request.user, change=change) + change["errors"] = [str(e)] + errors.append(change) + return errors + + def add_to_community_library( + self, channel_id, channel_version, categories, country_codes + ): + # The change to add a channel to the community library can only + # be created server-side, so in theory we should not be getting + # malformed requests here. However, just to be safe, we still + # do basic checks. + + channel = self.get_edit_queryset().get(pk=channel_id) + countries = Country.objects.filter(code__in=country_codes) + + if not channel.public: + raise ValidationError( + "Only public channels can be added to the community library" + ) + if channel_version <= 0 or channel_version > channel.version: + raise ValidationError("Invalid channel version") + + export_channel_to_kolibri_public( + channel_id=channel_id, + channel_version=channel_version, + public=False, # Community library + categories=categories, + countries=countries, + ) + @action( detail=True, methods=["get"], diff --git a/contentcuration/contentcuration/viewsets/community_library_submission.py b/contentcuration/contentcuration/viewsets/community_library_submission.py index 163ab50e77..d99786fd67 100644 --- a/contentcuration/contentcuration/viewsets/community_library_submission.py +++ b/contentcuration/contentcuration/viewsets/community_library_submission.py @@ -9,9 +9,11 @@ from contentcuration.constants import ( community_library_submission as community_library_submission_constants, ) +from contentcuration.models import Change from contentcuration.models import Channel from contentcuration.models import CommunityLibrarySubmission from contentcuration.models import Country +from contentcuration.tasks import apply_channel_changes_task from contentcuration.utils.pagination import ValuesViewsetCursorPagination from contentcuration.viewsets.base import BulkListSerializer from contentcuration.viewsets.base import BulkModelSerializer @@ -20,6 +22,9 @@ from contentcuration.viewsets.base import RESTDestroyModelMixin from contentcuration.viewsets.base import RESTUpdateModelMixin from contentcuration.viewsets.common import UserFilteredPrimaryKeyRelatedField +from contentcuration.viewsets.sync.utils import ( + generate_added_to_community_library_event, +) from contentcuration.viewsets.user import IsAdminUser @@ -232,10 +237,29 @@ def _mark_previous_pending_submissions_as_superseded(self, submission): channel_version__lt=submission.channel_version, ).update(status=community_library_submission_constants.STATUS_SUPERSEDED) + def _add_to_community_library(self, submission): + country_codes = sorted(country.code for country in submission.countries.all()) + + Change.create_change( + generate_added_to_community_library_event( + key=submission.channel.id, + channel_version=submission.channel_version, + categories=submission.categories, + country_codes=country_codes, + ), + created_by_id=submission.resolved_by_id, + ) + apply_channel_changes_task.fetch_or_enqueue( + submission.resolved_by, + channel_id=submission.channel.id, + ) + @action( methods=["post"], detail=True, serializer_class=CommunityLibrarySubmissionResolveSerializer, + url_name="resolve", + url_path="resolve", ) def resolve(self, request, pk=None): instance = self.get_edit_object() @@ -250,5 +274,6 @@ def resolve(self, request, pk=None): if submission.status == community_library_submission_constants.STATUS_APPROVED: self._mark_previous_pending_submissions_as_superseded(submission) + self._add_to_community_library(submission) return Response(self.serialize_object()) diff --git a/contentcuration/contentcuration/viewsets/sync/base.py b/contentcuration/contentcuration/viewsets/sync/base.py index 1d4be37c6b..2379e5b0c2 100644 --- a/contentcuration/contentcuration/viewsets/sync/base.py +++ b/contentcuration/contentcuration/viewsets/sync/base.py @@ -12,6 +12,7 @@ from contentcuration.viewsets.contentnode import PrerequisitesUpdateHandler from contentcuration.viewsets.file import FileViewSet from contentcuration.viewsets.invitation import InvitationViewSet +from contentcuration.viewsets.sync.constants import ADDED_TO_COMMUNITY_LIBRARY from contentcuration.viewsets.sync.constants import ASSESSMENTITEM from contentcuration.viewsets.sync.constants import BOOKMARK from contentcuration.viewsets.sync.constants import CHANNEL @@ -98,6 +99,7 @@ def get_change_type(obj): DEPLOYED: "deploy_from_changes", UPDATED_DESCENDANTS: "update_descendants_from_changes", PUBLISHED_NEXT: "publish_next_from_changes", + ADDED_TO_COMMUNITY_LIBRARY: "add_to_community_library_from_changes", } diff --git a/contentcuration/contentcuration/viewsets/sync/constants.py b/contentcuration/contentcuration/viewsets/sync/constants.py index 54091c9203..e45d30a3e0 100644 --- a/contentcuration/contentcuration/viewsets/sync/constants.py +++ b/contentcuration/contentcuration/viewsets/sync/constants.py @@ -9,6 +9,7 @@ DEPLOYED = 8 UPDATED_DESCENDANTS = 9 PUBLISHED_NEXT = 10 +ADDED_TO_COMMUNITY_LIBRARY = 11 ALL_CHANGES = set( @@ -23,6 +24,13 @@ DEPLOYED, UPDATED_DESCENDANTS, PUBLISHED_NEXT, + ADDED_TO_COMMUNITY_LIBRARY, + ] +) + +SERVER_ONLY_CHANGES = set( + [ + ADDED_TO_COMMUNITY_LIBRARY, ] ) diff --git a/contentcuration/contentcuration/viewsets/sync/endpoint.py b/contentcuration/contentcuration/viewsets/sync/endpoint.py index 6825833823..fcc5bed4fb 100644 --- a/contentcuration/contentcuration/viewsets/sync/endpoint.py +++ b/contentcuration/contentcuration/viewsets/sync/endpoint.py @@ -20,6 +20,7 @@ from contentcuration.tasks import apply_user_changes_task from contentcuration.viewsets.sync.constants import CHANNEL from contentcuration.viewsets.sync.constants import CREATED +from contentcuration.viewsets.sync.constants import SERVER_ONLY_CHANGES CHANGE_RETURN_LIMIT = 200 @@ -72,7 +73,11 @@ def handle_changes(self, request): # Changes that cannot be made disallowed_changes = [] for c in changes: - if c.get("channel_id") is None and c.get("user_id") == request.user.id: + if c.get("type") in SERVER_ONLY_CHANGES: + disallowed_changes.append(c) + elif ( + c.get("channel_id") is None and c.get("user_id") == request.user.id + ): user_only_changes.append(c) elif c.get("channel_id") in allowed_ids: channel_changes.append(c) diff --git a/contentcuration/contentcuration/viewsets/sync/utils.py b/contentcuration/contentcuration/viewsets/sync/utils.py index a0073ce731..e7df210926 100644 --- a/contentcuration/contentcuration/viewsets/sync/utils.py +++ b/contentcuration/contentcuration/viewsets/sync/utils.py @@ -3,6 +3,7 @@ from django.conf import settings from contentcuration.utils.sentry import report_exception +from contentcuration.viewsets.sync.constants import ADDED_TO_COMMUNITY_LIBRARY from contentcuration.viewsets.sync.constants import ALL_TABLES from contentcuration.viewsets.sync.constants import CHANNEL from contentcuration.viewsets.sync.constants import CONTENTNODE @@ -104,6 +105,25 @@ def generate_publish_next_event(key, version_notes="", language=None): return event +def generate_added_to_community_library_event( + key, + channel_version, + categories=None, + country_codes=None, +): + event = _generate_event( + key, + CHANNEL, + ADDED_TO_COMMUNITY_LIBRARY, + channel_id=key, + user_id=None, + ) + event["channel_version"] = channel_version + event["categories"] = categories or dict() + event["country_codes"] = country_codes or list() + return event + + def log_sync_exception(e, user=None, change=None, changes=None): # Capture exception and report, but allow sync # to complete properly. diff --git a/contentcuration/kolibri_public/management/commands/export_channels_to_kolibri_public.py b/contentcuration/kolibri_public/management/commands/export_channels_to_kolibri_public.py index 3a0aee8fcd..07080fb849 100644 --- a/contentcuration/kolibri_public/management/commands/export_channels_to_kolibri_public.py +++ b/contentcuration/kolibri_public/management/commands/export_channels_to_kolibri_public.py @@ -1,23 +1,14 @@ import logging import os -import shutil -import tempfile from datetime import datetime from datetime import timedelta -from django.conf import settings -from django.core.files.storage import default_storage as storage -from django.core.management import call_command from django.core.management.base import BaseCommand from django.db.models import F from django.db.models import Q from django.utils import timezone -from kolibri_content.apps import KolibriContentConfig -from kolibri_content.models import ChannelMetadata as ExportedChannelMetadata -from kolibri_content.router import get_active_content_database -from kolibri_content.router import using_content_database from kolibri_public.models import ChannelMetadata -from kolibri_public.utils.mapper import ChannelMapper +from kolibri_public.utils import export_channel_to_kolibri_public from contentcuration.models import Channel from contentcuration.models import User @@ -55,7 +46,7 @@ def handle(self, *args, **options): count = 0 for channel_id in ids_to_export: try: - self._export_channel(channel_id) + export_channel_to_kolibri_public(channel_id) count += 1 except FileNotFoundError: logger.warning( @@ -71,31 +62,6 @@ def handle(self, *args, **options): ) logger.info("Successfully put {} channels into kolibri_public".format(count)) - def _export_channel(self, channel_id): - logger.info("Putting channel {} into kolibri_public".format(channel_id)) - db_location = os.path.join( - settings.DB_ROOT, "{id}.sqlite3".format(id=channel_id) - ) - with storage.open(db_location) as storage_file: - with tempfile.NamedTemporaryFile(suffix=".sqlite3") as db_file: - shutil.copyfileobj(storage_file, db_file) - db_file.seek(0) - with using_content_database(db_file.name): - # Run migration to handle old content databases published prior to current fields being added. - call_command( - "migrate", - app_label=KolibriContentConfig.label, - database=get_active_content_database(), - ) - channel = ExportedChannelMetadata.objects.get(id=channel_id) - logger.info( - "Found channel {} for id: {} mapping now".format( - channel.name, channel_id - ) - ) - mapper = ChannelMapper(channel) - mapper.run() - def _republish_problem_channels(self): twenty_19 = datetime(year=2019, month=1, day=1) five_minutes = timedelta(minutes=5) diff --git a/contentcuration/kolibri_public/migrations/0007_new_channel_metadata.py b/contentcuration/kolibri_public/migrations/0007_new_channel_metadata.py new file mode 100644 index 0000000000..cfa24e91d2 --- /dev/null +++ b/contentcuration/kolibri_public/migrations/0007_new_channel_metadata.py @@ -0,0 +1,26 @@ +# Generated by Django 3.2.24 on 2025-07-22 10:38 +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + + dependencies = [ + ("contentcuration", "0157_alter_communitylibrarysubmission_resolution_reason"), + ("kolibri_public", "0006_auto_20250417_1516"), + ] + + operations = [ + migrations.AddField( + model_name="channelmetadata", + name="categories", + field=models.JSONField(blank=True, null=True), + ), + migrations.AddField( + model_name="channelmetadata", + name="countries", + field=models.ManyToManyField( + related_name="public_channels", to="contentcuration.Country" + ), + ), + ] diff --git a/contentcuration/kolibri_public/models.py b/contentcuration/kolibri_public/models.py index c0056d9cf9..35b3ede984 100644 --- a/contentcuration/kolibri_public/models.py +++ b/contentcuration/kolibri_public/models.py @@ -7,6 +7,7 @@ from mptt.managers import TreeManager from mptt.querysets import TreeQuerySet +from contentcuration.models import Country from contentcuration.models import Language @@ -104,6 +105,8 @@ class ChannelMetadata(base_models.ChannelMetadata): ) order = models.PositiveIntegerField(default=0, null=True, blank=True) public = models.BooleanField() + categories = models.JSONField(null=True, blank=True) + countries = models.ManyToManyField(Country, related_name="public_channels") class MPTTTreeIDManager(models.Model): diff --git a/contentcuration/kolibri_public/tests/test_export_channels_to_kolibri_public.py b/contentcuration/kolibri_public/tests/test_export_channels_to_kolibri_public.py new file mode 100644 index 0000000000..1a8fdba131 --- /dev/null +++ b/contentcuration/kolibri_public/tests/test_export_channels_to_kolibri_public.py @@ -0,0 +1,152 @@ +import os.path +import shutil +import tempfile +import uuid +from unittest import mock + +from django.conf import settings +from django.core.files.storage import FileSystemStorage +from django.core.management import call_command +from django.db import models as django_models +from django.test import TestCase +from kolibri_content.apps import KolibriContentConfig +from kolibri_content.fields import UUIDField +from kolibri_content.models import ChannelMetadata as ExportedChannelMetadata +from kolibri_content.router import get_active_content_database +from kolibri_content.router import using_content_database +from kolibri_public.utils.export_channel_to_kolibri_public import ( + export_channel_to_kolibri_public, +) +from mixer.backend.django import GenFactory +from mixer.backend.django import Mixer + +from contentcuration.models import Country + + +class CustomizedMixer(Mixer): + """Slightly modified Mixer that works correctly with the active + content database and with UUIDField. + """ + + def __init__(self, *args, **kwargs): + mixer_factory = GenFactory() + mixer_factory.generators[UUIDField] = mixer_factory.generators[ + django_models.UUIDField + ] + + return super().__init__(*args, factory=mixer_factory, **kwargs) + + def postprocess(self, target): + if self.params.get("commit"): + # Not sure why the `force_insert` is needed, but using the + # mixer causes "Save with update_fields did not affect any rows" error + # if this is not specified + target.save(using=get_active_content_database(), force_insert=True) + + return target + + +class ExportTestCase(TestCase): + def setUp(self): + super().setUp() + + self._temp_directory_ctx = tempfile.TemporaryDirectory() + test_db_root_dir = self._temp_directory_ctx.__enter__() + + storage = FileSystemStorage(location=test_db_root_dir) + + self._storage_patch_ctx = mock.patch( + "kolibri_public.utils.export_channel_to_kolibri_public.storage", + new=storage, + ) + self._storage_patch_ctx.__enter__() + + os.makedirs(os.path.join(test_db_root_dir, settings.DB_ROOT), exist_ok=True) + + self.channel_id = uuid.UUID(int=42).hex + self.channel_version = 1 + + db_path = os.path.join( + test_db_root_dir, + settings.DB_ROOT, + f"{self.channel_id}-{self.channel_version}.sqlite3", + ) + open(db_path, "w").close() + + with using_content_database(db_path): + call_command( + "migrate", + app_label=KolibriContentConfig.label, + database=get_active_content_database(), + ) + + mixer = CustomizedMixer() + self.exported_channel_metadata = mixer.blend( + ExportedChannelMetadata, + id=self.channel_id, + version=self.channel_version, + ) + + self.db_path_without_version = os.path.join( + test_db_root_dir, settings.DB_ROOT, f"{self.channel_id}.sqlite3" + ) + shutil.copyfile(db_path, self.db_path_without_version) + + def tearDown(self): + self._temp_directory_ctx.__exit__(None, None, None) + self._storage_patch_ctx.__exit__(None, None, None) + + super().tearDown() + + @mock.patch("kolibri_public.utils.export_channel_to_kolibri_public.ChannelMapper") + def test_export_channel_to_kolibri_public__existing_version( + self, mock_channel_mapper + ): + categories = { + "Category1": True, + "Category2": True, + } + country1 = Country.objects.create(code="C1", name="Country 1") + country2 = Country.objects.create(code="C2", name="Country 2") + countries = [country1, country2] + + export_channel_to_kolibri_public( + channel_id=self.channel_id, + channel_version=1, + public=True, + categories=categories, + countries=countries, + ) + + mock_channel_mapper.assert_called_once_with( + channel=self.exported_channel_metadata, + channel_version=1, + public=True, + categories=categories, + countries=countries, + ) + mock_channel_mapper.return_value.run.assert_called_once_with() + + @mock.patch("kolibri_public.utils.export_channel_to_kolibri_public.ChannelMapper") + def test_export_channel_to_kolibri_public__without_version( + self, mock_channel_mapper + ): + export_channel_to_kolibri_public( + channel_id=self.channel_id, + ) + + mock_channel_mapper.assert_called_once_with( + channel=self.exported_channel_metadata, + channel_version=None, + public=True, + categories=None, + countries=None, + ) + mock_channel_mapper.return_value.run.assert_called_once_with() + + def test_export_channel_to_kolibri_public__bad_version(self): + with self.assertRaises(FileNotFoundError): + export_channel_to_kolibri_public( + channel_id=self.channel_id, + channel_version=2, + ) diff --git a/contentcuration/kolibri_public/tests/test_mapper.py b/contentcuration/kolibri_public/tests/test_mapper.py index d938233e63..edefafcd25 100644 --- a/contentcuration/kolibri_public/tests/test_mapper.py +++ b/contentcuration/kolibri_public/tests/test_mapper.py @@ -1,6 +1,9 @@ +import datetime import os import tempfile +from unittest import mock +import pytz from django.core.management import call_command from django.test import TestCase from kolibri_content import models as kolibri_content_models @@ -14,6 +17,7 @@ from le_utils.constants import content_kinds from contentcuration.models import Channel +from contentcuration.models import Country from contentcuration.tests.testdata import user @@ -28,16 +32,24 @@ def overrides(self): kolibri_public_models.LocalFile: { "available": True, }, + kolibri_public_models.ChannelMetadata: { + "last_updated": self.dummy_date, + }, } - @classmethod - def setUpClass(cls): - super(ChannelMapperTest, cls).setUpClass() + def setUp(self): + super().setUp() call_command("loadconstants") - _, cls.tempdb = tempfile.mkstemp(suffix=".sqlite3") + _, self.tempdb = tempfile.mkstemp(suffix=".sqlite3") admin_user = user() - with using_content_database(cls.tempdb): + self.dummy_date = datetime.datetime(2020, 1, 1, 0, 0, 0, tzinfo=pytz.utc) + self._date_patcher = mock.patch( + "kolibri_public.utils.annotation.timezone.now", return_value=self.dummy_date + ) + self._date_patcher.start() + + with using_content_database(self.tempdb): call_command( "migrate", "content", @@ -52,23 +64,20 @@ def setUpClass(cls): }, ) builder.insert_into_default_db() - cls.source_root = kolibri_content_models.ContentNode.objects.get( + self.source_root = kolibri_content_models.ContentNode.objects.get( id=builder.root_node["id"] ) - cls.channel = kolibri_content_models.ChannelMetadata.objects.get( + self.channel = kolibri_content_models.ChannelMetadata.objects.get( id=builder.channel["id"] ) contentcuration_channel = Channel.objects.create( actor_id=admin_user.id, - id=cls.channel.id, - name=cls.channel.name, + id=self.channel.id, + name=self.channel.name, public=True, ) contentcuration_channel.main_tree.published = True contentcuration_channel.main_tree.save() - cls.mapper = ChannelMapper(cls.channel) - cls.mapper.run() - cls.mapped_root = cls.mapper.mapped_root def _assert_model(self, source, mapped, Model): for field in Model._meta.fields: @@ -79,7 +88,11 @@ def _assert_model(self, source, mapped, Model): self.overrides[Model][column], getattr(mapped, column) ) else: - self.assertEqual(getattr(source, column), getattr(mapped, column)) + self.assertEqual( + getattr(source, column), + getattr(mapped, column), + f"Mismatch in model {Model.__name__}, column {column}", + ) def _assert_node(self, source, mapped): """ @@ -133,6 +146,10 @@ def _recurse_and_assert(self, sources, mappeds, recursion_depth=0): def test_map(self): with using_content_database(self.tempdb): + self.mapper = ChannelMapper(self.channel) + self.mapper.run() + self.mapped_root = self.mapper.mapped_root + self._recurse_and_assert([self.source_root], [self.mapped_root]) self._assert_model( self.channel, @@ -142,6 +159,10 @@ def test_map(self): def test_map_replace(self): with using_content_database(self.tempdb): + self.mapper = ChannelMapper(self.channel) + self.mapper.run() + self.mapped_root = self.mapper.mapped_root + mapper = ChannelMapper(self.channel) mapper.run() self._recurse_and_assert([self.source_root], [mapper.mapped_root]) @@ -151,10 +172,105 @@ def test_map_replace(self): kolibri_public_models.ChannelMetadata, ) - @classmethod - def tearDownClass(cls): + def test_categories__none_provided(self): + with using_content_database(self.tempdb): + kolibri_content_models.ContentNode.objects.filter( + channel_id=self.channel.id, + ).update(categories=None) + + mapper = ChannelMapper(self.channel) + mapper.run() + + self.assertEqual(mapper.mapped_channel.categories, {}) + + def test_categories__only_provided(self): + with using_content_database(self.tempdb): + kolibri_content_models.ContentNode.objects.filter( + channel_id=self.channel.id, + ).update(categories=None) + + categories = {"Category1": True, "Category2": True} + mapper = ChannelMapper(self.channel, categories=categories) + mapper.run() + + self.assertEqual(mapper.mapped_channel.categories, categories) + + def test_categories__only_on_content_nodes(self): + with using_content_database(self.tempdb): + source_contentnodes_queryset = ( + kolibri_content_models.ContentNode.objects.filter( + channel_id=self.channel.id, + ) + ) + contentnode1 = source_contentnodes_queryset[0] + contentnode2 = source_contentnodes_queryset[1] + + source_contentnodes_queryset.update(categories=None) + contentnode1.categories = "Category1,Category2" + contentnode2.categories = "Category3" + contentnode1.save() + contentnode2.save() + + mapper = ChannelMapper(self.channel) + mapper.run() + + self.assertEqual( + mapper.mapped_channel.categories, + {"Category1": True, "Category2": True, "Category3": True}, + ) + + def test_categories__both_provided_and_on_content_nodes(self): + with using_content_database(self.tempdb): + source_contentnodes_queryset = ( + kolibri_content_models.ContentNode.objects.filter( + channel_id=self.channel.id, + ) + ) + contentnode1 = source_contentnodes_queryset[0] + contentnode2 = source_contentnodes_queryset[1] + + source_contentnodes_queryset.update(categories=None) + contentnode1.categories = "Category1,Category2" + contentnode2.categories = "Category3" + contentnode1.save() + contentnode2.save() + + categories = {"Category3": True, "Category4": True} + mapper = ChannelMapper(self.channel, categories=categories) + mapper.run() + + self.assertEqual( + mapper.mapped_channel.categories, + { + "Category1": True, + "Category2": True, + "Category3": True, + "Category4": True, + }, + ) + + def test_countries__none_provided(self): + with using_content_database(self.tempdb): + mapper = ChannelMapper(self.channel) + mapper.run() + + self.assertEqual(mapper.mapped_channel.countries.count(), 0) + + def test_countries__provided(self): + with using_content_database(self.tempdb): + country1 = Country.objects.create(code="C1", name="Country 1") + country2 = Country.objects.create(code="C2", name="Country 2") + + countries = [country1, country2] + mapper = ChannelMapper(self.channel, countries=countries) + mapper.run() + + self.assertCountEqual(mapper.mapped_channel.countries.all(), countries) + + def tearDown(self): # Clean up datbase connection after the test - cleanup_content_database_connection(cls.tempdb) - super(ChannelMapperTest, cls).tearDownClass() - if os.path.exists(cls.tempdb): - os.remove(cls.tempdb) + self._date_patcher.stop() + cleanup_content_database_connection(self.tempdb) + super().tearDown() + if os.path.exists(self.tempdb): + os.remove(self.tempdb) diff --git a/contentcuration/kolibri_public/utils/annotation.py b/contentcuration/kolibri_public/utils/annotation.py index 7295c97e39..c2889282ae 100644 --- a/contentcuration/kolibri_public/utils/annotation.py +++ b/contentcuration/kolibri_public/utils/annotation.py @@ -1,11 +1,12 @@ """ -Functions in here are the subset of annotation functions from Kolibri related to channel metadata. +Functions in here are a modified subset of annotation functions from Kolibri related to channel metadata. https://github.com/learningequality/kolibri/blob/caec91dd2da5617adfb50332fb698068248e8e47/kolibri/core/content/utils/annotation.py#L731 """ -import datetime +from itertools import chain from django.db.models import Q from django.db.models import Sum +from django.utils import timezone from kolibri_public.models import ChannelMetadata from kolibri_public.models import ContentNode from kolibri_public.models import LocalFile @@ -14,18 +15,29 @@ from contentcuration.models import Channel -def set_channel_metadata_fields(channel_id, public=None): +def set_channel_metadata_fields( + channel_id, + channel_version=None, + public=None, + categories=None, + countries=None, +): # Remove unneeded db_lock channel = ChannelMetadata.objects.get(id=channel_id) calculate_published_size(channel) calculate_total_resource_count(channel) calculate_included_languages(channel) + calculate_included_categories(channel, categories) calculate_next_order(channel, public=public) # Add this to ensure we keep this up to date. - channel.last_updated = datetime.datetime.now() + channel.last_updated = timezone.now() if public is not None: channel.public = public + if channel_version is not None: + channel.version = channel_version + if countries is not None: + channel.countries.set(countries) channel.save() @@ -67,6 +79,28 @@ def calculate_included_languages(channel): channel.included_languages.add(*list(languages)) +def calculate_included_categories(channel, categories): + content_nodes = ContentNode.objects.filter( + channel_id=channel.id, available=True + ).exclude(categories=None) + + categories_comma_separated_lists = content_nodes.values_list( + "categories", flat=True + ) + contentnode_categories = set( + chain.from_iterable( + ( + categories_comma_separated_list.split(",") + for categories_comma_separated_list in categories_comma_separated_lists + ) + ) + ) + categories_dict = {category: True for category in contentnode_categories} + categories_dict.update(categories or {}) + channel.categories = categories_dict + channel.save() + + def calculate_next_order(channel, public=False): # This has been edited from the source Kolibri, in order # to make the order match given by the public channel API on Studio. diff --git a/contentcuration/kolibri_public/utils/export_channel_to_kolibri_public.py b/contentcuration/kolibri_public/utils/export_channel_to_kolibri_public.py new file mode 100644 index 0000000000..4ba442dc87 --- /dev/null +++ b/contentcuration/kolibri_public/utils/export_channel_to_kolibri_public.py @@ -0,0 +1,60 @@ +import logging +import os +import shutil +import tempfile + +from django.conf import settings +from django.core.files.storage import default_storage as storage +from django.core.management import call_command +from kolibri_content.apps import KolibriContentConfig +from kolibri_content.models import ChannelMetadata as ExportedChannelMetadata +from kolibri_content.router import get_active_content_database +from kolibri_content.router import using_content_database +from kolibri_public.utils.mapper import ChannelMapper + + +logger = logging.getLogger(__file__) + + +def export_channel_to_kolibri_public( + channel_id, + channel_version=None, + public=True, + categories=None, + countries=None, +): + logger.info("Putting channel {} into kolibri_public".format(channel_id)) + + if channel_version is not None: + db_filename = "{id}-{version}.sqlite3".format( + id=channel_id, version=channel_version + ) + else: + db_filename = "{id}.sqlite3".format(id=channel_id) + db_location = os.path.join(settings.DB_ROOT, db_filename) + + with storage.open(db_location) as storage_file: + with tempfile.NamedTemporaryFile(suffix=".sqlite3") as db_file: + shutil.copyfileobj(storage_file, db_file) + db_file.seek(0) + with using_content_database(db_file.name): + # Run migration to handle old content databases published prior to current fields being added. + call_command( + "migrate", + app_label=KolibriContentConfig.label, + database=get_active_content_database(), + ) + channel = ExportedChannelMetadata.objects.get(id=channel_id) + logger.info( + "Found channel {} for id: {} mapping now".format( + channel.name, channel_id + ) + ) + mapper = ChannelMapper( + channel=channel, + channel_version=channel_version, + public=public, + categories=categories, + countries=countries, + ) + mapper.run() diff --git a/contentcuration/kolibri_public/utils/mapper.py b/contentcuration/kolibri_public/utils/mapper.py index 01dc1f0726..f9258bfb7d 100644 --- a/contentcuration/kolibri_public/utils/mapper.py +++ b/contentcuration/kolibri_public/utils/mapper.py @@ -19,9 +19,19 @@ class ChannelMapper(object): Foreign Keyed from the root ContentNode. """ - def __init__(self, channel, public=True): + def __init__( + self, + channel, + channel_version=None, + public=True, + categories=None, + countries=None, + ): self.channel = channel + self.channel_version = channel_version self.public = public + self.categories = categories + self.countries = countries @property def overrides(self): @@ -57,7 +67,19 @@ def run(self): annotate_label_bitmasks(self.mapped_root.get_descendants(include_self=True)) # Rather than set the ancestors fields after mapping, like it is done in Kolibri # here we set it during mapping as we are already recursing through the tree. - set_channel_metadata_fields(self.mapped_channel.id, public=self.public) + + set_channel_metadata_fields( + self.mapped_channel.id, + channel_version=self.channel_version, + public=self.public, + categories=self.categories, + countries=self.countries, + ) + + # Refreshing this is needed, because otherwise the fields set in the + # set_channel_metadata_fields function will not be reflected in the + # self.mapped_channel object. + self.mapped_channel.refresh_from_db() def _map_model(self, source, Model): properties = {} From 93f281045859ee7649df454b12409e88bcf75455 Mon Sep 17 00:00:00 2001 From: Jakub Komarek Date: Wed, 30 Jul 2025 16:07:15 +0200 Subject: [PATCH 034/472] Fix migration dependency --- .../kolibri_public/migrations/0007_new_channel_metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contentcuration/kolibri_public/migrations/0007_new_channel_metadata.py b/contentcuration/kolibri_public/migrations/0007_new_channel_metadata.py index cfa24e91d2..1e2144d6b3 100644 --- a/contentcuration/kolibri_public/migrations/0007_new_channel_metadata.py +++ b/contentcuration/kolibri_public/migrations/0007_new_channel_metadata.py @@ -6,7 +6,7 @@ class Migration(migrations.Migration): dependencies = [ - ("contentcuration", "0157_alter_communitylibrarysubmission_resolution_reason"), + ("contentcuration", "0156_communitylibrarysubmission_admin_fields"), ("kolibri_public", "0006_auto_20250417_1516"), ] From 4043ca0164a4c536073ea28b1acd0f688bd12989 Mon Sep 17 00:00:00 2001 From: habibayman Date: Fri, 1 Aug 2025 06:43:31 +0300 Subject: [PATCH 035/472] tests(texteditor): write tests for loading custom markdown --- .../TipTapEditor/utils/markdown.js | 8 +- .../TipTapEditor/__tests__/markdown.spec.js | 123 ++++++++++++++++++ 2 files changed, 129 insertions(+), 2 deletions(-) create mode 100644 contentcuration/contentcuration/frontend/shared/views/TipTapEditor/__tests__/markdown.spec.js diff --git a/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/utils/markdown.js b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/utils/markdown.js index 0ac3204ade..0b823625f2 100644 --- a/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/utils/markdown.js +++ b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/utils/markdown.js @@ -95,8 +95,12 @@ export function preprocessMarkdown(markdown) { // 2. The permanentSrc is just the checksum + extension. const permanentSrc = `${params.checksum}.${params.extension}`; - // 3. Create an tag with the REAL display URL in `src`. - return `${params.alt}`; + // 3. Create attributes string for width and height only if they exist + const widthAttr = params.width ? ` width="${params.width}"` : ''; + const heightAttr = params.height ? ` height="${params.height}"` : ''; + + // 4. Create an tag with the REAL display URL in `src`. + return `${params.alt}`; }); processedMarkdown = processedMarkdown.replace(MATH_REGEX, match => { diff --git a/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/__tests__/markdown.spec.js b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/__tests__/markdown.spec.js new file mode 100644 index 0000000000..520e056666 --- /dev/null +++ b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/__tests__/markdown.spec.js @@ -0,0 +1,123 @@ +// eslint-disable-next-line import/namespace +import { marked } from 'marked'; +import { preprocessMarkdown } from '../TipTapEditor/utils/markdown'; +import { storageUrl } from '../../../vuex/file/utils'; + +// Mock the dependencies at the top level +jest.mock('marked', () => ({ + marked: jest.fn(str => str), +})); +jest.mock('../../../vuex/file/utils', () => ({ + storageUrl: jest.fn((checksum, ext) => `/content/storage/${checksum}.${ext}`), +})); + +describe('preprocessMarkdown', () => { + beforeEach(() => { + marked.mockClear(); + storageUrl.mockClear(); + }); + + // 1: Legacy Checksum Images + // Old images did not add dimensions unless a resize was used. + describe('Legacy Image Handling', () => { + it('should convert a legacy image with dimensions into a full tag', () => { + const legacyMd = '![some alt text](${CONTENTSTORAGE}/checksum.png =100x200)'; + const expectedHtml = + 'some alt text'; + expect(preprocessMarkdown(legacyMd)).toBe(expectedHtml); + }); + + it('should detect & convert a legacy image WITHOUT dimensions', () => { + const legacyMd = '![No Dims](${☣ CONTENTSTORAGE}/file.jpg)'; + const expectedHtml = + 'No Dims'; + expect(preprocessMarkdown(legacyMd)).toBe(expectedHtml); + }); + + it('should process multiple legacy images correctly', () => { + const mixedMd = [ + '![first](${CONTENTSTORAGE}/image1.jpg =100x100)', + 'Some text', + '![second](${CONTENTSTORAGE}/image2.png)', + ].join('\n'); + + const expectedHtml = [ + 'first', + 'Some text', + 'second', + ].join('\n'); + + const result = preprocessMarkdown(mixedMd); + + expect(result).toBe(expectedHtml); + expect(storageUrl).toHaveBeenCalledWith('image1', 'jpg'); + expect(storageUrl).toHaveBeenCalledWith('image2', 'png'); + }); + }); + + // 2: Data URL / Blob URL Images + describe('Data URL Image Handling', () => { + it('should convert a data URL image with dimensions into an tag', () => { + const dataUrlMd = '![Data](data:image/png;base64,iVBORw0KGgo= =300x400)'; + const expectedHtml = + 'Data'; + expect(preprocessMarkdown(dataUrlMd)).toBe(expectedHtml); + }); + }); + + // 3: Math Formulas + describe('Math Formula Handling', () => { + it('should convert a simple math formula into a tag', () => { + const mathMd = 'Some text $$x^2$$ and more text.'; + const expectedHtml = 'Some text and more text.'; + expect(preprocessMarkdown(mathMd)).toBe(expectedHtml); + }); + + it('should handle math formulas with spaces', () => { + const mathMd = 'Formula: $$ x + y = z $$'; + const expectedHtml = 'Formula: '; + expect(preprocessMarkdown(mathMd)).toBe(expectedHtml); + }); + }); + + // 4: Standard Markdown Passthrough + describe('Standard Markdown Passthrough', () => { + it('should pass non-custom syntax through to the marked library', () => { + const standardMd = 'Here is **bold** and a [link](url).'; + preprocessMarkdown(standardMd); + // We test the *interaction*, not the result (the library). + expect(marked).toHaveBeenCalled(); + expect(marked).toHaveBeenCalledWith(standardMd); + }); + }); + + // 5: Mixed Content + describe('Mixed Content Handling', () => { + it('should correctly process a string with legacy images, data URLs, math, and standard markdown', () => { + const mixedMd = + 'Legacy: ![](${CONTENTSTORAGE}/file.png =10x10)\nData: ![alt](data:image/gif;base64,R0lGODlhAQABAAD/ACwAAAAAAQABAAACADs=)\nFormula: $$E=mc^2$$'; + const expectedHtml = + 'Legacy: \nData: alt\nFormula: '; + + const result = preprocessMarkdown(mixedMd); + expect(result).toBe(expectedHtml); + }); + }); + + // 6: Edge Cases + describe('Edge Case Handling', () => { + it('should return an empty string for null, undefined, or empty string input', () => { + expect(preprocessMarkdown(null)).toBe(''); + expect(preprocessMarkdown(undefined)).toBe(''); + expect(preprocessMarkdown('')).toBe(''); + }); + + it('should handle very long input without crashing', () => { + const longMd = 'Text '.repeat(10000) + '$$x^2$$'; + expect(() => preprocessMarkdown(longMd)).not.toThrow(); + + const result = preprocessMarkdown(longMd); + expect(result).toContain(''); + }); + }); +}); From c54c68825eb5cbed4716de5cefcd16c48581b737 Mon Sep 17 00:00:00 2001 From: habibayman Date: Fri, 1 Aug 2025 09:34:32 +0300 Subject: [PATCH 036/472] fix(markdown srializer): add depth to handle nested lists --- .../TipTapEditor/utils/markdownSerializer.js | 43 ++++++++++++------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/utils/markdownSerializer.js b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/utils/markdownSerializer.js index 261a59372e..d859771eb9 100644 --- a/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/utils/markdownSerializer.js +++ b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/utils/markdownSerializer.js @@ -44,7 +44,7 @@ export const createCustomMarkdownSerializer = editor => { return text; }; - const serializeNode = (node, listNumber = null) => { + const serializeNode = (node, listNumber = null, depth = 0) => { if (!node || !node.type) { return; } @@ -57,7 +57,7 @@ export const createCustomMarkdownSerializer = editor => { const child = node.content.content[i]; if (child) { if (i > 0) result += '\n\n'; - serializeNode(child); + serializeNode(child, null, depth); } } } @@ -68,7 +68,7 @@ export const createCustomMarkdownSerializer = editor => { for (let i = 0; i < node.content.size; i++) { const child = node.content.content[i]; if (child) { - serializeNode(child); + serializeNode(child, null, depth); } } } @@ -81,7 +81,7 @@ export const createCustomMarkdownSerializer = editor => { for (let i = 0; i < node.content.size; i++) { const child = node.content.content[i]; if (child) { - serializeNode(child); + serializeNode(child, null, depth); } } } @@ -106,7 +106,7 @@ export const createCustomMarkdownSerializer = editor => { for (let i = 0; i < node.content.size; i++) { const child = node.content.content[i]; if (child) { - serializeNode(child); + serializeNode(child, null, depth); } } } @@ -117,7 +117,7 @@ export const createCustomMarkdownSerializer = editor => { for (let i = 0; i < node.content.size; i++) { const child = node.content.content[i]; if (child) { - serializeNode(child, 'bullet'); + serializeNode(child, 'bullet', depth); if (i < node.content.size - 1) result += '\n'; } } @@ -127,22 +127,27 @@ export const createCustomMarkdownSerializer = editor => { for (let i = 0; i < node.content.size; i++) { const child = node.content.content[i]; if (child) { - serializeNode(child, i + 1); + serializeNode(child, i + 1, depth); if (i < node.content.size - 1) result += '\n'; } } break; - case 'listItem': + case 'listItem': { + // Add indentation for nested lists + const indent = ' '.repeat(depth); + // Use the passed listNumber parameter if (listNumber === 'bullet') { - result += '- '; + result += indent + '- '; } else if (typeof listNumber === 'number') { - result += `${listNumber}. `; + result += indent + `${listNumber}. `; } // Process list item content properly if (node.content && node.content.size > 0) { + let hasProcessedFirstParagraph = false; + for (let i = 0; i < node.content.size; i++) { const child = node.content.content[i]; if (child && child.type) { @@ -152,17 +157,25 @@ export const createCustomMarkdownSerializer = editor => { for (let j = 0; j < child.content.size; j++) { const grandchild = child.content.content[j]; if (grandchild) { - serializeNode(grandchild); + serializeNode(grandchild, null, depth); } } } + hasProcessedFirstParagraph = true; + } else if (child.type.name === 'bulletList' || child.type.name === 'orderedList') { + // Handle nested lists + if (hasProcessedFirstParagraph) { + result += '\n'; + } + serializeNode(child, null, depth + 1); } else { - serializeNode(child); + serializeNode(child, null, depth); } } } } break; + } case 'blockquote': result += '> '; @@ -170,7 +183,7 @@ export const createCustomMarkdownSerializer = editor => { for (let i = 0; i < node.content.size; i++) { const child = node.content.content[i]; if (child) { - serializeNode(child); + serializeNode(child, null, depth); } } } @@ -195,13 +208,13 @@ export const createCustomMarkdownSerializer = editor => { default: // Fallback: try to process children if (node.content) { - node.content.forEach(child => serializeNode(child)); + node.content.forEach(child => serializeNode(child, null, depth)); } break; } }; - serializeNode(doc, true); + serializeNode(doc, null, 0); return result.trim(); }; }; From 3ee55305c8ae537528af1de4b147e23ba5188ddd Mon Sep 17 00:00:00 2001 From: habibayman Date: Fri, 1 Aug 2025 18:10:51 +0300 Subject: [PATCH 037/472] tests(texteditor): write tests for custom markdown serializer --- .../__tests__/markdownSerializer.spec.js | 322 ++++++++++++++++++ 1 file changed, 322 insertions(+) create mode 100644 contentcuration/contentcuration/frontend/shared/views/TipTapEditor/__tests__/markdownSerializer.spec.js diff --git a/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/__tests__/markdownSerializer.spec.js b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/__tests__/markdownSerializer.spec.js new file mode 100644 index 0000000000..72f56ebeb0 --- /dev/null +++ b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/__tests__/markdownSerializer.spec.js @@ -0,0 +1,322 @@ +// Import the function we are testing +import { createCustomMarkdownSerializer } from '../TipTapEditor/utils/markdownSerializer'; + +// Mock the IMAGE_PLACEHOLDER to match what our tests expect +jest.mock('../TipTapEditor/utils/markdown', () => { + const originalModule = jest.requireActual('../TipTapEditor/utils/markdown'); + return { + ...originalModule, + IMAGE_PLACEHOLDER: '${CONTENTSTORAGE}', + paramsToImageMd: jest.fn(params => { + const alt = params.alt || ''; + const src = params.permanentSrc ? `\${CONTENTSTORAGE}/${params.permanentSrc}` : params.src; + const dimensions = params.width && params.height ? ` =${params.width}x${params.height}` : ''; + return `![${alt}](${src}${dimensions})`; + }), + }; +}); + +// mock editor object with a proper ProseMirror-like structure +const createMockEditor = docContent => { + // Create a mock node structure that mimics ProseMirror nodes + const createNode = node => { + const result = { ...node }; + + // Add required properties expected by the serializer + if (result.content) { + const contentArray = result.content.map(createNode); + result.content = { + size: contentArray.length, + content: contentArray, + // Add forEach to support iteration in the serializer + forEach: function (callback) { + contentArray.forEach(callback); + }, + }; + } + + if (result.type && typeof result.type === 'string') { + result.type = { name: result.type }; + } + + if (result.type && result.type.name === 'text' && result.text) { + result.textContent = result.text; + } + + // Process marks to make them compatible with the serializer + if (result.marks && Array.isArray(result.marks)) { + result.marks = result.marks.map(mark => { + if (typeof mark.type === 'string') { + return { ...mark, type: { name: mark.type } }; + } + return mark; + }); + } + + return result; + }; + + // Create the document structure + const doc = createNode({ + type: 'doc', + content: docContent, + }); + + return { + state: { + doc: doc, + }, + }; +}; + +describe('createCustomMarkdownSerializer', () => { + // 1: Custom Block Nodes + describe('Custom Block Node Serialization', () => { + it('should serialize a legacy image node using its permanentSrc', () => { + const docContent = [ + { + type: 'image', + attrs: { + src: '/content/storage/c/h/checksum.png', + permanentSrc: 'checksum.png', + alt: 'Legacy Cat', + width: '150', + height: '100', + }, + }, + ]; + const mockEditor = createMockEditor(docContent); + const getMarkdown = createCustomMarkdownSerializer(mockEditor); + + const expectedMd = '![Legacy Cat](${CONTENTSTORAGE}/checksum.png =150x100)'; + expect(getMarkdown()).toBe(expectedMd); + }); + + it('should serialize a new, unsaved data: URL image', () => { + const dataUrl = 'data:image/png;base64,iVBORw0KGgo='; + const docContent = [ + { + type: 'image', + attrs: { + src: dataUrl, + permanentSrc: null, // No permanent source yet + alt: 'New Cat', + width: '80', + height: '60', + }, + }, + ]; + const mockEditor = createMockEditor(docContent); + const getMarkdown = createCustomMarkdownSerializer(mockEditor); + + const expectedMd = `![New Cat](${dataUrl} =80x60)`; + expect(getMarkdown()).toBe(expectedMd); + }); + + it('should serialize a node correctly', () => { + const docContent = [ + { + type: 'small', + content: [{ type: 'text', text: 'This is small text.' }], + }, + ]; + const mockEditor = createMockEditor(docContent); + const getMarkdown = createCustomMarkdownSerializer(mockEditor); + + const expectedMd = 'This is small text.'; + expect(getMarkdown()).toBe(expectedMd); + }); + }); + + // 2: Custom Inline Nodes + describe('Custom Inline Node Serialization', () => { + it('should serialize a math node correctly', () => { + const docContent = [ + { + type: 'paragraph', + content: [ + { type: 'text', text: 'The formula is ' }, + { type: 'math', attrs: { latex: 'E=mc^2' } }, + { type: 'text', text: '.' }, + ], + }, + ]; + const mockEditor = createMockEditor(docContent); + const getMarkdown = createCustomMarkdownSerializer(mockEditor); + + const expectedMd = 'The formula is $$E=mc^2$$.'; + expect(getMarkdown()).toBe(expectedMd); + }); + }); + + // 3: Standard Block Nodes + describe('Standard Block Node Serialization', () => { + it('should serialize headings with the correct level', () => { + const docContent = [ + { + type: 'heading', + attrs: { level: 2 }, + content: [{ type: 'text', text: 'My Subtitle' }], + }, + ]; + const mockEditor = createMockEditor(docContent); + const getMarkdown = createCustomMarkdownSerializer(mockEditor); + expect(getMarkdown()).toBe('## My Subtitle'); + }); + + it('should serialize nested lists correctly', () => { + const docContent = [ + { + type: 'bulletList', + content: [ + { + type: 'listItem', + content: [ + { type: 'paragraph', content: [{ type: 'text', text: 'Item 1' }] }, + { + type: 'bulletList', + content: [ + { + type: 'listItem', + content: [ + { type: 'paragraph', content: [{ type: 'text', text: 'Nested 1.1' }] }, + ], + }, + ], + }, + ], + }, + { + type: 'listItem', + content: [{ type: 'paragraph', content: [{ type: 'text', text: 'Item 2' }] }], + }, + ], + }, + ]; + const mockEditor = createMockEditor(docContent); + const getMarkdown = createCustomMarkdownSerializer(mockEditor); + expect(getMarkdown()).toBe('- Item 1\n - Nested 1.1\n- Item 2'); + }); + + it('should place two newlines between block elements', () => { + const docContent = [ + { type: 'paragraph', content: [{ type: 'text', text: 'First paragraph.' }] }, + { type: 'paragraph', content: [{ type: 'text', text: 'Second paragraph.' }] }, + ]; + const mockEditor = createMockEditor(docContent); + const getMarkdown = createCustomMarkdownSerializer(mockEditor); + expect(getMarkdown()).toBe('First paragraph.\n\nSecond paragraph.'); + }); + }); + + // 4: Inline Formatting (Marks) + describe('Mark Serialization', () => { + it('should serialize a text node with multiple marks correctly', () => { + const docContent = [ + { + type: 'paragraph', + content: [ + { + type: 'text', + text: 'bold and italic', + marks: [{ type: 'bold' }, { type: 'italic' }], + }, + ], + }, + ]; + const mockEditor = createMockEditor(docContent); + const getMarkdown = createCustomMarkdownSerializer(mockEditor); + // The order of marks can vary, but typically it's inside-out + expect(getMarkdown()).toBe('*' + '**bold and italic**' + '*'); + }); + + it('should serialize an underlined node with a tag', () => { + const docContent = [ + { + type: 'paragraph', + content: [ + { + type: 'text', + text: 'underlined', + marks: [{ type: 'underline' }], + }, + ], + }, + ]; + const mockEditor = createMockEditor(docContent); + const getMarkdown = createCustomMarkdownSerializer(mockEditor); + expect(getMarkdown()).toBe('underlined'); + }); + + it('should serialize a link node correctly', () => { + const docContent = [ + { + type: 'paragraph', + content: [ + { + type: 'text', + text: 'Google', + marks: [{ type: 'link', attrs: { href: 'https://google.com' } }], + }, + ], + }, + ]; + const mockEditor = createMockEditor(docContent); + const getMarkdown = createCustomMarkdownSerializer(mockEditor); + expect(getMarkdown()).toBe('[Google](https://google.com)'); + }); + }); + + // 5: Edge cases + describe('Structural and Edge Case Serialization', () => { + it('should return an empty string for an empty document', () => { + const docContent = []; + const mockEditor = createMockEditor(docContent); + const getMarkdown = createCustomMarkdownSerializer(mockEditor); + expect(getMarkdown()).toBe(''); + }); + + it('should serialize very deep nested lists correctly', () => { + // Helper function to create nested list structure + const createNestedList = (levels, currentLevel = 1) => { + const listItem = { + type: 'listItem', + content: [ + { type: 'paragraph', content: [{ type: 'text', text: `Level ${currentLevel}` }] }, + ], + }; + + if (currentLevel < levels) { + listItem.content.push({ + type: 'bulletList', + content: [createNestedList(levels, currentLevel + 1)], + }); + } + + return listItem; + }; + + const docContent = [ + { + type: 'bulletList', + content: [ + createNestedList(5), + { + type: 'listItem', + content: [ + { type: 'paragraph', content: [{ type: 'text', text: 'Back to Level 1' }] }, + ], + }, + ], + }, + ]; + + const mockEditor = createMockEditor(docContent); + const getMarkdown = createCustomMarkdownSerializer(mockEditor); + + const expectedMd = + '- Level 1\n - Level 2\n - Level 3\n - Level 4\n - Level 5\n- Back to Level 1'; + expect(getMarkdown()).toBe(expectedMd); + }); + }); +}); From 48a0bc4e8467be664a6c74ef3ec30747e5faeaa3 Mon Sep 17 00:00:00 2001 From: habibayman Date: Fri, 1 Aug 2025 18:42:48 +0300 Subject: [PATCH 038/472] refactor(texteditor)[image]: use lodash debounce --- .../components/image/ImageNodeView.vue | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/components/image/ImageNodeView.vue b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/components/image/ImageNodeView.vue index b2e78c3387..9e5e3c4f8a 100644 --- a/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/components/image/ImageNodeView.vue +++ b/contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/components/image/ImageNodeView.vue @@ -67,6 +67,7 @@ import { defineComponent, ref, computed, onUnmounted, onMounted, watch } from 'vue'; import { NodeViewWrapper } from '@tiptap/vue-2'; + import _ from 'lodash'; export default defineComponent({ name: 'ImageNodeView', @@ -80,7 +81,14 @@ const naturalAspectRatio = ref(null); const minWidth = 50; const compactThreshold = 200; - let debounceTimer = null; + + // Create debounced version of saveSize function + const debouncedSaveSize = _.debounce(() => { + props.updateAttributes({ + width: width.value, + height: height.value, + }); + }, 500); // (to work with undo/redo) Watch for external changes to the node's width and height watch( @@ -209,8 +217,7 @@ width.value = clampedWidth; height.value = calculateProportionalHeight(clampedWidth); - clearTimeout(debounceTimer); - debounceTimer = setTimeout(saveSize, 500); + debouncedSaveSize(); }; const removeImage = () => { @@ -235,7 +242,8 @@ }); onUnmounted(() => { - clearTimeout(debounceTimer); + // Cancel any pending debounced calls + debouncedSaveSize.cancel(); }); return { From 9b33a874f76e0df8dcd4f19bf05f3d98021326e5 Mon Sep 17 00:00:00 2001 From: taoerman Date: Fri, 1 Aug 2025 16:45:19 -0700 Subject: [PATCH 039/472] Update channel logic to support publishing channel draft update tests for publish draft using main tree Enhanced publish functionality to support three publish types fixed fix frontend tests --- .../pages/StagingTreePage/index.vue | 4 +- .../vuex/currentChannel/actions.js | 4 + .../shared/data/__tests__/changes.spec.js | 5 +- .../frontend/shared/data/changes.js | 5 +- .../frontend/shared/data/resources.js | 5 +- .../frontend/shared/data/serverSync.js | 2 +- .../tests/test_exportchannel.py | 113 +++++++++++++++++- .../contentcuration/tests/viewsets/base.py | 4 +- .../tests/viewsets/test_channel.py | 4 +- .../contentcuration/utils/publish.py | 40 ++++--- .../contentcuration/viewsets/channel.py | 24 ++-- .../contentcuration/viewsets/sync/utils.py | 3 +- 12 files changed, 169 insertions(+), 44 deletions(-) diff --git a/contentcuration/contentcuration/frontend/channelEdit/pages/StagingTreePage/index.vue b/contentcuration/contentcuration/frontend/channelEdit/pages/StagingTreePage/index.vue index 6c7240e25f..612a870f7a 100644 --- a/contentcuration/contentcuration/frontend/channelEdit/pages/StagingTreePage/index.vue +++ b/contentcuration/contentcuration/frontend/channelEdit/pages/StagingTreePage/index.vue @@ -552,7 +552,7 @@ ...mapActions('currentChannel', [ 'loadCurrentChannelStagingDiff', 'deployCurrentChannel', - 'publishDraftChannel', + 'publishStagingChannel', 'reloadCurrentChannelStagingDiff', ]), ...mapActions('currentChannel', { loadCurrentChannel: 'loadChannel' }), @@ -646,7 +646,7 @@ this.displayPublishDraftDialog = false; this.isPublishingDraft = true; - this.publishDraftChannel() + this.publishStagingChannel() .then(() => { this.isPublishingDraft = false; this.showSnackbar({ diff --git a/contentcuration/contentcuration/frontend/channelEdit/vuex/currentChannel/actions.js b/contentcuration/contentcuration/frontend/channelEdit/vuex/currentChannel/actions.js index df5f528cd4..7659c0630d 100644 --- a/contentcuration/contentcuration/frontend/channelEdit/vuex/currentChannel/actions.js +++ b/contentcuration/contentcuration/frontend/channelEdit/vuex/currentChannel/actions.js @@ -59,6 +59,10 @@ export function publishDraftChannel(context) { return Channel.publishDraft(context.state.currentChannelId); } +export function publishStagingChannel(context) { + return Channel.publishDraft(context.state.currentChannelId, true); +} + export function channelLanguageExistsInResources(context) { const channelId = context.state.currentChannelId; return Channel.languageExistsInResources(channelId); diff --git a/contentcuration/contentcuration/frontend/shared/data/__tests__/changes.spec.js b/contentcuration/contentcuration/frontend/shared/data/__tests__/changes.spec.js index 51b69efe8b..9ea28f3e54 100644 --- a/contentcuration/contentcuration/frontend/shared/data/__tests__/changes.spec.js +++ b/contentcuration/contentcuration/frontend/shared/data/__tests__/changes.spec.js @@ -257,6 +257,9 @@ describe('Change Types', () => { const change = new PublishedNextChange({ key: '1', table: TABLE_NAMES.CHANNEL, + use_staging_tree: false, + version_notes: 'Test version notes', + language: 'en', source: CLIENTID, }); const rev = await change.saveChange(); @@ -264,7 +267,7 @@ describe('Change Types', () => { expect(persistedChange).toEqual({ rev, channel_id: change.key, - ...pick(change, ['type', 'key', 'table', 'source']), + ...pick(change, ['type', 'key', 'table', 'use_staging_tree', 'version_notes', 'language', 'source']), }); }); diff --git a/contentcuration/contentcuration/frontend/shared/data/changes.js b/contentcuration/contentcuration/frontend/shared/data/changes.js index 8c7f02d49a..a59e06ffeb 100644 --- a/contentcuration/contentcuration/frontend/shared/data/changes.js +++ b/contentcuration/contentcuration/frontend/shared/data/changes.js @@ -439,7 +439,7 @@ export class PublishedChange extends Change { } export class PublishedNextChange extends Change { - constructor(fields) { + constructor({ use_staging_tree, version_notes, language, ...fields }) { fields.type = CHANGE_TYPES.PUBLISHED_NEXT; super(fields); if (this.table !== TABLE_NAMES.CHANNEL) { @@ -447,6 +447,9 @@ export class PublishedNextChange extends Change { `${this.changeType} is only supported by ${TABLE_NAMES.CHANNEL} table but ${this.table} was passed instead`, ); } + this.setAndValidateBoolean(use_staging_tree, 'use_staging_tree'); + this.setAndValidateIsDefined(version_notes, 'version_notes'); + this.setAndValidateIsDefined(language, 'language'); this.setChannelAndUserId({ id: this.key }); } } diff --git a/contentcuration/contentcuration/frontend/shared/data/resources.js b/contentcuration/contentcuration/frontend/shared/data/resources.js index 17e3431cb7..b55245ffbd 100644 --- a/contentcuration/contentcuration/frontend/shared/data/resources.js +++ b/contentcuration/contentcuration/frontend/shared/data/resources.js @@ -1234,9 +1234,12 @@ export const Channel = new CreateModelResource({ }); }, - publishDraft(id) { + publishDraft(id, use_staging_tree=false, version_notes="") { const change = new PublishedNextChange({ key: id, + use_staging_tree: use_staging_tree, + version_notes: version_notes, + language: currentLanguage, table: this.tableName, source: CLIENTID, }); diff --git a/contentcuration/contentcuration/frontend/shared/data/serverSync.js b/contentcuration/contentcuration/frontend/shared/data/serverSync.js index 72c8c60e4c..0015766058 100644 --- a/contentcuration/contentcuration/frontend/shared/data/serverSync.js +++ b/contentcuration/contentcuration/frontend/shared/data/serverSync.js @@ -36,7 +36,7 @@ const ChangeTypeMapFields = { 'excluded_descendants', ]), [CHANGE_TYPES.PUBLISHED]: commonFields.concat(['version_notes', 'language']), - [CHANGE_TYPES.PUBLISHED_NEXT]: commonFields, + [CHANGE_TYPES.PUBLISHED_NEXT]: commonFields.concat(['use_staging_tree', 'version_notes', 'language']), [CHANGE_TYPES.SYNCED]: commonFields.concat([ 'titles_and_descriptions', 'resource_details', diff --git a/contentcuration/contentcuration/tests/test_exportchannel.py b/contentcuration/contentcuration/tests/test_exportchannel.py index 95d75e7886..c7c3097a4c 100644 --- a/contentcuration/contentcuration/tests/test_exportchannel.py +++ b/contentcuration/contentcuration/tests/test_exportchannel.py @@ -885,6 +885,7 @@ def run_publish_channel(self): send_email=False, progress_tracker=None, language="fr", + is_draft_version=True, use_staging_tree=True, ) @@ -894,11 +895,9 @@ def test_none_staging_tree(self): with self.assertRaises(NoneContentNodeTreeError): self.run_publish_channel() - def test_staging_tree_published(self): - self.assertFalse(self.content_channel.staging_tree.published) + def test_staging_tree_not_published_for_draft(self): self.run_publish_channel() - self.content_channel.refresh_from_db() - self.assertTrue(self.content_channel.staging_tree.published) + self.assertFalse(self.content_channel.staging_tree.published) def test_next_version_exported(self): self.run_publish_channel() @@ -928,6 +927,7 @@ def test_staging_tree_used_for_publish(self): self.admin_user.id, True, progress_tracker=None, + is_draft_version=True, use_staging_tree=True, ) set_active_content_database(self.tempdb) @@ -944,3 +944,108 @@ def test_staging_tree_used_for_publish(self): set_active_content_database(None) if os.path.exists(self.tempdb): os.remove(self.tempdb) + +class PublishDraftUsingMainTreeTestCase(StudioTestCase): + @classmethod + def setUpClass(cls): + super(PublishDraftUsingMainTreeTestCase, cls).setUpClass() + cls.patch_copy_db = patch("contentcuration.utils.publish.save_export_database") + cls.mock_save_export = cls.patch_copy_db.start() + + @classmethod + def tearDownClass(cls): + super(PublishDraftUsingMainTreeTestCase, cls).tearDownClass() + cls.patch_copy_db.stop() + + def setUp(self): + super(PublishDraftUsingMainTreeTestCase, self).setUp() + + self.channel_version = 3 + self.incomplete_video_in_main = "Incomplete video in main tree" + self.complete_video_in_main = "Complete video in main tree" + + self.content_channel = channel() + self.content_channel.version = self.channel_version + self.content_channel.save() + + # Incomplete node in main_tree should be excluded. + new_node = create_node( + {"kind_id": "video", "title": self.incomplete_video_in_main, "children": []} + ) + new_node.complete = False + new_node.parent = self.content_channel.main_tree + new_node.published = False + new_node.save() + + # Complete node in main_tree should be included. + new_node = create_node( + {"kind_id": "video", "title": self.complete_video_in_main, "children": []} + ) + new_node.complete = True + new_node.parent = self.content_channel.main_tree + new_node.published = False + new_node.save() + + def run_publish_channel(self): + publish_channel( + self.admin_user.id, + self.content_channel.id, + version_notes="", + force=False, + force_exercises=False, + send_email=False, + progress_tracker=None, + language="fr", + is_draft_version=True, + use_staging_tree=False, + ) + + def test_next_version_exported(self): + self.run_publish_channel() + self.mock_save_export.assert_called_with( + self.content_channel.id, + "next", + True, + ) + + def test_main_tree_not_impacted(self): + self.assertFalse(self.content_channel.main_tree.published) + self.run_publish_channel() + self.content_channel.refresh_from_db() + self.assertFalse(self.content_channel.main_tree.published) + + def test_channel_version_not_incremented(self): + self.assertEqual(self.content_channel.version, self.channel_version) + self.run_publish_channel() + self.content_channel.refresh_from_db() + self.assertEqual(self.content_channel.version, self.channel_version) + + def test_main_tree_used_for_publish(self): + set_channel_icon_encoding(self.content_channel) + self.tempdb = create_content_database( + self.content_channel, + True, + self.admin_user.id, + True, + progress_tracker=None, + is_draft_version=True, + use_staging_tree=False, + ) + set_active_content_database(self.tempdb) + + nodes = kolibri_models.ContentNode.objects.all() + self.assertEqual(nodes.filter(title=self.incomplete_video_in_main).count(), 0) + self.assertEqual(nodes.filter(title=self.complete_video_in_main).count(), 1) + + cleanup_content_database_connection(self.tempdb) + set_active_content_database(None) + if os.path.exists(self.tempdb): + os.remove(self.tempdb) + + def test_only_next_file_created(self): + self.mock_save_export.reset_mock() + self.run_publish_channel() + self.assertEqual(self.mock_save_export.call_count, 1) + call_args = self.mock_save_export.call_args + self.assertEqual(call_args[0][1], "next") + self.assertEqual(call_args[0][2], True) diff --git a/contentcuration/contentcuration/tests/viewsets/base.py b/contentcuration/contentcuration/tests/viewsets/base.py index 617d23bb26..23cea48f53 100644 --- a/contentcuration/contentcuration/tests/viewsets/base.py +++ b/contentcuration/contentcuration/tests/viewsets/base.py @@ -94,8 +94,8 @@ def generate_publish_channel_event(channel_id): return event -def generate_publish_next_event(channel_id): - event = base_generate_publish_next_event(channel_id) +def generate_publish_next_event(channel_id, use_staging_tree=False): + event = base_generate_publish_next_event(channel_id, use_staging_tree=use_staging_tree) event["rev"] = random.randint(1, 10000000) return event diff --git a/contentcuration/contentcuration/tests/viewsets/test_channel.py b/contentcuration/contentcuration/tests/viewsets/test_channel.py index 067e442ede..3bf8cf86d3 100644 --- a/contentcuration/contentcuration/tests/viewsets/test_channel.py +++ b/contentcuration/contentcuration/tests/viewsets/test_channel.py @@ -492,7 +492,7 @@ def test_publish_next(self): self.assertEqual(response.status_code, 200) modified_channel = models.Channel.objects.get(id=channel.id) - self.assertEqual(modified_channel.staging_tree.published, True) + self.assertEqual(modified_channel.staging_tree.published, False) def test_publish_next_with_incomplete_staging_tree(self): channel = testdata.channel() @@ -507,7 +507,7 @@ def test_publish_next_with_incomplete_staging_tree(self): channel.save() self.assertEqual(channel.staging_tree.published, False) - response = self.sync_changes([generate_publish_next_event(channel.id)]) + response = self.sync_changes([generate_publish_next_event(channel.id, use_staging_tree=True)]) self.assertEqual(response.status_code, 200) self.assertTrue( diff --git a/contentcuration/contentcuration/utils/publish.py b/contentcuration/contentcuration/utils/publish.py index 841f440134..fbd4b0651a 100644 --- a/contentcuration/contentcuration/utils/publish.py +++ b/contentcuration/contentcuration/utils/publish.py @@ -146,18 +146,22 @@ def create_content_database( user_id, force_exercises, progress_tracker=None, + is_draft_version=False, use_staging_tree=False, ): """ :type progress_tracker: contentcuration.utils.celery.ProgressTracker|None """ + if not is_draft_version and use_staging_tree: + raise ValueError("Staging tree is only supported for draft versions") + # increment the channel version - if not use_staging_tree and not force: + if not is_draft_version and not force: raise_if_nodes_are_all_unchanged(channel) fh, tempdb = tempfile.mkstemp(suffix=".sqlite3") with using_content_database(tempdb): - if not use_staging_tree and not channel.main_tree.publishing: + if not is_draft_version and not channel.main_tree.publishing: channel.mark_publishing(user_id) call_command( @@ -183,11 +187,11 @@ def create_content_database( progress_tracker.track(90) map_prerequisites(base_tree) # Need to save as version being published, not current version - version = "next" if use_staging_tree else channel.version + 1 + version = "next" if is_draft_version else channel.version + 1 save_export_database( channel.pk, version, - use_staging_tree, + is_draft_version, ) if channel.public: mapper = ChannelMapper(kolibri_channel) @@ -1127,14 +1131,14 @@ def mark_all_nodes_as_published(tree): logging.info("Marked all nodes as published.") -def save_export_database(channel_id, version, use_staging_tree=False): +def save_export_database(channel_id, version, is_draft_version=False): logging.debug("Saving export database") current_export_db_location = get_active_content_database() target_paths = [ os.path.join(settings.DB_ROOT, "{}-{}.sqlite3".format(channel_id, version)) ] # Only create non-version path if not using the staging tree - if not use_staging_tree: + if not is_draft_version: target_paths.append( os.path.join(settings.DB_ROOT, "{id}.sqlite3".format(id=channel_id)) ) @@ -1297,6 +1301,7 @@ def publish_channel( # noqa: C901 send_email=False, progress_tracker=None, language=settings.LANGUAGE_CODE, + is_draft_version=False, use_staging_tree=False, ): """ @@ -1317,23 +1322,23 @@ def publish_channel( # noqa: C901 user_id, force_exercises, progress_tracker=progress_tracker, - use_staging_tree=use_staging_tree, + is_draft_version=is_draft_version, + use_staging_tree=use_staging_tree, ) add_tokens_to_channel(channel) - if not use_staging_tree: + if not is_draft_version: increment_channel_version(channel) sync_contentnode_and_channel_tsvectors(channel_id=channel.id) mark_all_nodes_as_published(base_tree) fill_published_fields(channel, version_notes) - - # Attributes not getting set for some reason, so just save it here - base_tree.publishing = False - base_tree.changed = False - base_tree.published = True - base_tree.save() + base_tree.publishing = False + base_tree.changed = False + base_tree.published = True + base_tree.save() + # Delete public channel cache. - if not use_staging_tree and channel.public: + if not is_draft_version and channel.public: delete_public_channel_cache_keys() if send_email: @@ -1355,8 +1360,9 @@ def publish_channel( # noqa: C901 finally: if kolibri_temp_db and os.path.exists(kolibri_temp_db): os.remove(kolibri_temp_db) - base_tree.publishing = False - base_tree.save() + if not is_draft_version: + base_tree.publishing = False + base_tree.save() elapsed = time.time() - start diff --git a/contentcuration/contentcuration/viewsets/channel.py b/contentcuration/contentcuration/viewsets/channel.py index 1fd6625030..eb80790c5c 100644 --- a/contentcuration/contentcuration/viewsets/channel.py +++ b/contentcuration/contentcuration/viewsets/channel.py @@ -606,28 +606,29 @@ def publish(self, pk, version_notes="", language=None): raise def publish_next_from_changes(self, changes): + errors = [] for publish in changes: try: - self.publish_next(publish["key"]) + self.publish_next( + publish["key"], + version_notes=publish.get("version_notes"), + language=publish.get("language"), + use_staging_tree=publish.get("use_staging_tree", False), + ) except Exception as e: log_sync_exception(e, user=self.request.user, change=publish) publish["errors"] = [str(e)] errors.append(publish) return errors - def publish_next(self, pk): + def publish_next(self, pk, version_notes="", language=None, use_staging_tree=False): logging.debug("Entering the publish staging channel endpoint") channel = self.get_edit_queryset().get(pk=pk) if channel.deleted: raise ValidationError("Cannot publish a deleted channel") - elif channel.staging_tree.publishing: - raise ValidationError("Channel staging tree is already publishing") - - channel.staging_tree.publishing = True - channel.staging_tree.save() with create_change_tracker( pk, CHANNEL, channel.id, self.request.user, "export-channel-staging-tree" @@ -636,8 +637,11 @@ def publish_next(self, pk): channel = publish_channel( self.request.user.pk, channel.id, + version_notes=version_notes, progress_tracker=progress_tracker, - use_staging_tree=True, + language=language, + is_draft_version=True, + use_staging_tree=use_staging_tree, ) Change.create_changes( [ @@ -653,12 +657,8 @@ def publish_next(self, pk): applied=True, ) except ChannelIncompleteError: - channel.staging_tree.publishing = False - channel.staging_tree.save() raise ValidationError("Channel is not ready to be published") except Exception: - channel.staging_tree.publishing = False - channel.staging_tree.save() raise def sync_from_changes(self, changes): diff --git a/contentcuration/contentcuration/viewsets/sync/utils.py b/contentcuration/contentcuration/viewsets/sync/utils.py index a0073ce731..5fa93c8a6f 100644 --- a/contentcuration/contentcuration/viewsets/sync/utils.py +++ b/contentcuration/contentcuration/viewsets/sync/utils.py @@ -97,10 +97,11 @@ def generate_update_descendants_event(key, mods, channel_id=None, user_id=None): return event -def generate_publish_next_event(key, version_notes="", language=None): +def generate_publish_next_event(key, version_notes="", language=None, use_staging_tree=False): event = _generate_event(key, CHANNEL, PUBLISHED_NEXT, key, None) event["version_notes"] = version_notes event["language"] = language + event["use_staging_tree"] = use_staging_tree return event From e6d9950bceb58e0e63ff3c5d63acbb34ed3f327c Mon Sep 17 00:00:00 2001 From: Jakub Komarek Date: Mon, 4 Aug 2025 16:02:18 +0200 Subject: [PATCH 040/472] Use change key instead of channel_id --- contentcuration/contentcuration/viewsets/channel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contentcuration/contentcuration/viewsets/channel.py b/contentcuration/contentcuration/viewsets/channel.py index 5f8a33ade9..a3cea25b35 100644 --- a/contentcuration/contentcuration/viewsets/channel.py +++ b/contentcuration/contentcuration/viewsets/channel.py @@ -777,7 +777,7 @@ def add_to_community_library_from_changes(self, changes): for change in changes: try: self.add_to_community_library( - channel_id=change["channel_id"], + channel_id=change["key"], channel_version=change["channel_version"], categories=change["categories"], country_codes=change["country_codes"], From 611ccd16d6c2912d17d8cdf1c79eea5c304a18d1 Mon Sep 17 00:00:00 2001 From: Jakub Komarek Date: Mon, 4 Aug 2025 16:34:55 +0200 Subject: [PATCH 041/472] Don't allow creating community library submissions for public channels --- contentcuration/contentcuration/viewsets/channel.py | 4 ++-- .../contentcuration/viewsets/community_library_submission.py | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/contentcuration/contentcuration/viewsets/channel.py b/contentcuration/contentcuration/viewsets/channel.py index a3cea25b35..adccaa4373 100644 --- a/contentcuration/contentcuration/viewsets/channel.py +++ b/contentcuration/contentcuration/viewsets/channel.py @@ -799,9 +799,9 @@ def add_to_community_library( channel = self.get_edit_queryset().get(pk=channel_id) countries = Country.objects.filter(code__in=country_codes) - if not channel.public: + if channel.public: raise ValidationError( - "Only public channels can be added to the community library" + "Public channels cannot be added to the community library" ) if channel_version <= 0 or channel_version > channel.version: raise ValidationError("Invalid channel version") diff --git a/contentcuration/contentcuration/viewsets/community_library_submission.py b/contentcuration/contentcuration/viewsets/community_library_submission.py index d99786fd67..8a19542b33 100644 --- a/contentcuration/contentcuration/viewsets/community_library_submission.py +++ b/contentcuration/contentcuration/viewsets/community_library_submission.py @@ -61,6 +61,11 @@ def create(self, validated_data): "unpublished channel." ) + if channel.public: + raise ValidationError( + "Cannot create a community library submission for a public channel." + ) + if not channel.editors.filter(id=user.id).exists(): raise ValidationError( "Only editors can create a community library " From 7fce8bcee2d3c85ea47084f65ce3afc7b408a3f0 Mon Sep 17 00:00:00 2001 From: Jakub Komarek Date: Mon, 4 Aug 2025 16:51:54 +0200 Subject: [PATCH 042/472] Fix tests to test with non-public channels --- .../tests/viewsets/test_channel.py | 2 +- .../test_community_library_submission.py | 25 ++++++++++++++++--- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/contentcuration/contentcuration/tests/viewsets/test_channel.py b/contentcuration/contentcuration/tests/viewsets/test_channel.py index 9ea043966c..bed1af5978 100644 --- a/contentcuration/contentcuration/tests/viewsets/test_channel.py +++ b/contentcuration/contentcuration/tests/viewsets/test_channel.py @@ -549,7 +549,7 @@ def test_process_added_to_community_library_change(self, mock_export_func): channel = testdata.channel() channel.version = 1 - channel.public = True + channel.public = False channel.save() categories = { diff --git a/contentcuration/contentcuration/tests/viewsets/test_community_library_submission.py b/contentcuration/contentcuration/tests/viewsets/test_community_library_submission.py index f0fff43647..5ff525c4fa 100644 --- a/contentcuration/contentcuration/tests/viewsets/test_community_library_submission.py +++ b/contentcuration/contentcuration/tests/viewsets/test_community_library_submission.py @@ -61,21 +61,21 @@ def setUp(self): self.country2 = testdata.country(name="Country 2", code="C2") self.channel_with_submission1 = testdata.channel() - self.channel_with_submission1.public = True + self.channel_with_submission1.public = False self.channel_with_submission1.version = 1 self.channel_with_submission1.editors.add(self.author_user) self.channel_with_submission1.editors.add(self.editor_user) self.channel_with_submission1.save() self.channel_with_submission2 = testdata.channel() - self.channel_with_submission2.public = True + self.channel_with_submission2.public = False self.channel_with_submission2.version = 1 self.channel_with_submission2.editors.add(self.author_user) self.channel_with_submission2.editors.add(self.editor_user) self.channel_with_submission2.save() self.channel_without_submission = testdata.channel() - self.channel_without_submission.public = True + self.channel_without_submission.public = False self.channel_without_submission.version = 1 self.channel_without_submission.editors.add(self.author_user) self.channel_without_submission.editors.add(self.editor_user) @@ -88,6 +88,13 @@ def setUp(self): self.unpublished_channel.editors.add(self.editor_user) self.unpublished_channel.save() + self.public_channel = testdata.channel() + self.public_channel.public = True + self.public_channel.version = 1 + self.public_channel.editors.add(self.author_user) + self.public_channel.editors.add(self.editor_user) + self.public_channel.save() + self.existing_submission1 = testdata.community_library_submission() self.existing_submission1.channel = self.channel_with_submission1 self.existing_submission1.author = self.author_user @@ -134,6 +141,18 @@ def test_create_submission__unpublished_channel(self): ) self.assertEqual(response.status_code, 400, response.content) + def test_create_submission__public_channel(self): + self.client.force_authenticate(user=self.editor_user) + submission = self.new_submission_metadata + submission["channel"] = self.public_channel.id + + response = self.client.post( + reverse("community-library-submission-list"), + submission, + format="json", + ) + self.assertEqual(response.status_code, 400, response.content) + def test_create_submission__explicit_channel_version(self): self.client.force_authenticate(user=self.editor_user) submission_metadata = self.new_submission_metadata From b8fee43669c2b6c1aca4f247bb017dd6dfa7cbb8 Mon Sep 17 00:00:00 2001 From: Jakub Komarek Date: Mon, 4 Aug 2025 18:30:51 +0200 Subject: [PATCH 043/472] Set submission as live when added to community library --- .../tests/viewsets/test_channel.py | 36 +++++++++++++++++-- .../contentcuration/viewsets/channel.py | 21 +++++++++++ 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/contentcuration/contentcuration/tests/viewsets/test_channel.py b/contentcuration/contentcuration/tests/viewsets/test_channel.py index bed1af5978..932789de55 100644 --- a/contentcuration/contentcuration/tests/viewsets/test_channel.py +++ b/contentcuration/contentcuration/tests/viewsets/test_channel.py @@ -11,7 +11,9 @@ from contentcuration import models from contentcuration import models as cc from contentcuration.constants import channel_history +from contentcuration.constants import community_library_submission from contentcuration.models import Change +from contentcuration.models import CommunityLibrarySubmission from contentcuration.models import ContentNode from contentcuration.models import Country from contentcuration.tasks import apply_channel_changes_task @@ -547,11 +549,27 @@ def test_process_added_to_community_library_change(self, mock_export_func): # Creating the change on the backend should be supported self.client.force_authenticate(self.admin_user) + editor_user = testdata.user("channel@editor.com") + channel = testdata.channel() - channel.version = 1 + channel.version = 2 channel.public = False + channel.editors.add(editor_user) channel.save() + current_live_submission = CommunityLibrarySubmission.objects.create( + channel=channel, + channel_version=1, + author=editor_user, + status=community_library_submission.STATUS_LIVE, + ) + new_submission = CommunityLibrarySubmission.objects.create( + channel=channel, + channel_version=2, + author=editor_user, + status=community_library_submission.STATUS_APPROVED, + ) + categories = { "category1": True, "category2": True, @@ -564,7 +582,7 @@ def test_process_added_to_community_library_change(self, mock_export_func): added_to_community_library_change = generate_added_to_community_library_event( key=channel.id, - channel_version=1, + channel_version=2, categories=categories, country_codes=country_codes, ) @@ -596,7 +614,7 @@ def test_process_added_to_community_library_change(self, mock_export_func): ], ) self.assertEqual(call_kwargs["channel_id"], channel.id) - self.assertEqual(call_kwargs["channel_version"], 1) + self.assertEqual(call_kwargs["channel_version"], 2) self.assertEqual(call_kwargs["categories"], categories) # The countries argument used when creating the mapper is in fact @@ -604,6 +622,18 @@ def test_process_added_to_community_library_change(self, mock_export_func): self.assertCountEqual(call_kwargs["countries"], countries) self.assertEqual(call_kwargs["public"], False) + # Check that the current submission became the live one + current_live_submission.refresh_from_db() + new_submission.refresh_from_db() + self.assertEqual( + current_live_submission.status, + community_library_submission.STATUS_APPROVED, + ) + self.assertEqual( + new_submission.status, + community_library_submission.STATUS_LIVE, + ) + class CRUDTestCase(StudioAPITestCase): @property diff --git a/contentcuration/contentcuration/viewsets/channel.py b/contentcuration/contentcuration/viewsets/channel.py index adccaa4373..7f4b3e8b5b 100644 --- a/contentcuration/contentcuration/viewsets/channel.py +++ b/contentcuration/contentcuration/viewsets/channel.py @@ -42,9 +42,13 @@ from search.utils import get_fts_search_query import contentcuration.models as models +from contentcuration.constants import ( + community_library_submission as community_library_submission_constants, +) from contentcuration.decorators import cache_no_user_data from contentcuration.models import Change from contentcuration.models import Channel +from contentcuration.models import CommunityLibrarySubmission from contentcuration.models import ContentNode from contentcuration.models import Country from contentcuration.models import File @@ -814,6 +818,23 @@ def add_to_community_library( countries=countries, ) + # Mark the submission corresponding to the channel version + # as the only live submission for the channel + CommunityLibrarySubmission.objects.filter( + channel_id=channel_id, + status=community_library_submission_constants.STATUS_LIVE, + ).update( + status=community_library_submission_constants.STATUS_APPROVED, + ) + + new_live_submission = CommunityLibrarySubmission.objects.get( + channel_id=channel_id, + channel_version=channel_version, + status=community_library_submission_constants.STATUS_APPROVED, + ) + new_live_submission.status = community_library_submission_constants.STATUS_LIVE + new_live_submission.save() + @action( detail=True, methods=["get"], From 6f5bdfbf8dd1759a4af1b5c4bc7ad398c724f68b Mon Sep 17 00:00:00 2001 From: Jakub Komarek Date: Mon, 4 Aug 2025 19:04:48 +0200 Subject: [PATCH 044/472] Fix handling partial submission update without countries --- .../test_community_library_submission.py | 24 ++++++++++++++++++- .../viewsets/community_library_submission.py | 5 ++-- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/contentcuration/contentcuration/tests/viewsets/test_community_library_submission.py b/contentcuration/contentcuration/tests/viewsets/test_community_library_submission.py index 5ff525c4fa..58c3ae5871 100644 --- a/contentcuration/contentcuration/tests/viewsets/test_community_library_submission.py +++ b/contentcuration/contentcuration/tests/viewsets/test_community_library_submission.py @@ -338,7 +338,7 @@ def test_update_submission__is_editor(self): ) self.assertEqual(response.status_code, 404, response.content) - def test_update_submission__is_admin(self): + def test_update_submission__is_admin__change_countries(self): self.client.force_authenticate(user=self.admin_user) response = self.client.put( reverse( @@ -356,6 +356,28 @@ def test_update_submission__is_admin(self): self.assertEqual(updated_submission.countries.count(), 1) self.assertEqual(updated_submission.countries.first().code, "C2") + def test_update_submission__is_admin__keep_countries(self): + self.client.force_authenticate(user=self.admin_user) + + updated_submission_metadata = self.updated_submission_metadata.copy() + updated_submission_metadata.pop("countries") + + response = self.client.put( + reverse( + "community-library-submission-detail", + args=[self.existing_submission1.id], + ), + updated_submission_metadata, + format="json", + ) + self.assertEqual(response.status_code, 200, response.content) + + updated_submission = CommunityLibrarySubmission.objects.get( + id=self.existing_submission1.id + ) + self.assertEqual(updated_submission.countries.count(), 1) + self.assertEqual(updated_submission.countries.first().code, "C1") + def test_update_submission__change_channel(self): self.client.force_authenticate(user=self.admin_user) submission_metadata = self.updated_submission_metadata diff --git a/contentcuration/contentcuration/viewsets/community_library_submission.py b/contentcuration/contentcuration/viewsets/community_library_submission.py index 8a19542b33..5ae7562b2f 100644 --- a/contentcuration/contentcuration/viewsets/community_library_submission.py +++ b/contentcuration/contentcuration/viewsets/community_library_submission.py @@ -93,8 +93,9 @@ def update(self, instance, validated_data): "a community library submission." ) - countries = validated_data.pop("countries", []) - instance.countries.set(countries) + if "countries" in validated_data: + countries = validated_data.pop("countries") + instance.countries.set(countries) return super().update(instance, validated_data) From 8a0235e53d57fd971a9b1cba3d9eb813249a6549 Mon Sep 17 00:00:00 2001 From: Jakub Komarek Date: Mon, 4 Aug 2025 19:24:00 +0200 Subject: [PATCH 045/472] Remove explicit channel_version argument for ChannelMapper --- contentcuration/kolibri_public/utils/annotation.py | 3 --- .../utils/export_channel_to_kolibri_public.py | 1 - contentcuration/kolibri_public/utils/mapper.py | 6 +++--- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/contentcuration/kolibri_public/utils/annotation.py b/contentcuration/kolibri_public/utils/annotation.py index c2889282ae..1c9a193511 100644 --- a/contentcuration/kolibri_public/utils/annotation.py +++ b/contentcuration/kolibri_public/utils/annotation.py @@ -17,7 +17,6 @@ def set_channel_metadata_fields( channel_id, - channel_version=None, public=None, categories=None, countries=None, @@ -34,8 +33,6 @@ def set_channel_metadata_fields( if public is not None: channel.public = public - if channel_version is not None: - channel.version = channel_version if countries is not None: channel.countries.set(countries) channel.save() diff --git a/contentcuration/kolibri_public/utils/export_channel_to_kolibri_public.py b/contentcuration/kolibri_public/utils/export_channel_to_kolibri_public.py index 4ba442dc87..d751bd6614 100644 --- a/contentcuration/kolibri_public/utils/export_channel_to_kolibri_public.py +++ b/contentcuration/kolibri_public/utils/export_channel_to_kolibri_public.py @@ -52,7 +52,6 @@ def export_channel_to_kolibri_public( ) mapper = ChannelMapper( channel=channel, - channel_version=channel_version, public=public, categories=categories, countries=countries, diff --git a/contentcuration/kolibri_public/utils/mapper.py b/contentcuration/kolibri_public/utils/mapper.py index f9258bfb7d..bd779ea6d2 100644 --- a/contentcuration/kolibri_public/utils/mapper.py +++ b/contentcuration/kolibri_public/utils/mapper.py @@ -22,13 +22,14 @@ class ChannelMapper(object): def __init__( self, channel, - channel_version=None, public=True, categories=None, countries=None, ): + # Note: The argument `channel` is an instance of `kolibri_content.models.ChannelMetadata,` + # which belongs to a specific channel version to be exported. Therefore, we do not + # need to explicitly pass the channel version as an argument here. self.channel = channel - self.channel_version = channel_version self.public = public self.categories = categories self.countries = countries @@ -70,7 +71,6 @@ def run(self): set_channel_metadata_fields( self.mapped_channel.id, - channel_version=self.channel_version, public=self.public, categories=self.categories, countries=self.countries, From 9152aeb2b8a1fa06c74453663ec4595abe3c644c Mon Sep 17 00:00:00 2001 From: Jakub Komarek Date: Mon, 4 Aug 2025 22:50:27 +0200 Subject: [PATCH 046/472] Fix tests expecting the `channel_version` argument --- .../tests/test_export_channels_to_kolibri_public.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/contentcuration/kolibri_public/tests/test_export_channels_to_kolibri_public.py b/contentcuration/kolibri_public/tests/test_export_channels_to_kolibri_public.py index 1a8fdba131..d0e22d4e2c 100644 --- a/contentcuration/kolibri_public/tests/test_export_channels_to_kolibri_public.py +++ b/contentcuration/kolibri_public/tests/test_export_channels_to_kolibri_public.py @@ -120,7 +120,6 @@ def test_export_channel_to_kolibri_public__existing_version( mock_channel_mapper.assert_called_once_with( channel=self.exported_channel_metadata, - channel_version=1, public=True, categories=categories, countries=countries, @@ -137,7 +136,6 @@ def test_export_channel_to_kolibri_public__without_version( mock_channel_mapper.assert_called_once_with( channel=self.exported_channel_metadata, - channel_version=None, public=True, categories=None, countries=None, From 550fe78fee720cc5d1b7ac45164507c74b774401 Mon Sep 17 00:00:00 2001 From: Jakub Komarek Date: Mon, 4 Aug 2025 22:51:12 +0200 Subject: [PATCH 047/472] Represent categories as a list after passing from changes --- .../tests/viewsets/test_channel.py | 2 +- .../contentcuration/viewsets/channel.py | 12 +++++++++++- contentcuration/kolibri_public/models.py | 2 ++ .../test_export_channels_to_kolibri_public.py | 5 +---- .../kolibri_public/tests/test_mapper.py | 15 +++++---------- .../kolibri_public/utils/annotation.py | 8 +++++--- .../utils/export_channel_to_kolibri_public.py | 1 + contentcuration/kolibri_public/utils/mapper.py | 3 +++ 8 files changed, 29 insertions(+), 19 deletions(-) diff --git a/contentcuration/contentcuration/tests/viewsets/test_channel.py b/contentcuration/contentcuration/tests/viewsets/test_channel.py index 932789de55..d0e1dd4fca 100644 --- a/contentcuration/contentcuration/tests/viewsets/test_channel.py +++ b/contentcuration/contentcuration/tests/viewsets/test_channel.py @@ -615,7 +615,7 @@ def test_process_added_to_community_library_change(self, mock_export_func): ) self.assertEqual(call_kwargs["channel_id"], channel.id) self.assertEqual(call_kwargs["channel_version"], 2) - self.assertEqual(call_kwargs["categories"], categories) + self.assertCountEqual(call_kwargs["categories"], categories.keys()) # The countries argument used when creating the mapper is in fact # not a list, but a QuerySet, but it contains the same elements diff --git a/contentcuration/contentcuration/viewsets/channel.py b/contentcuration/contentcuration/viewsets/channel.py index 7f4b3e8b5b..4a51b37319 100644 --- a/contentcuration/contentcuration/viewsets/channel.py +++ b/contentcuration/contentcuration/viewsets/channel.py @@ -795,6 +795,10 @@ def add_to_community_library_from_changes(self, changes): def add_to_community_library( self, channel_id, channel_version, categories, country_codes ): + # Note: The `categories` field should contain a _dict_, with the category IDs as keys + # and `True` as a value. This is to match the representation + # of sets in the changes architecture. + # The change to add a channel to the community library can only # be created server-side, so in theory we should not be getting # malformed requests here. However, just to be safe, we still @@ -810,11 +814,17 @@ def add_to_community_library( if channel_version <= 0 or channel_version > channel.version: raise ValidationError("Invalid channel version") + # Because of changes architecture, the categories are passed as a dict + # with the category IDs as keys and `True` as a value. At this point, + # we are no longer working with changes, so we switch to the more + # convenient representation as a list. + categories_list = [key for key, val in categories.items() if val] + export_channel_to_kolibri_public( channel_id=channel_id, channel_version=channel_version, public=False, # Community library - categories=categories, + categories=categories_list, countries=countries, ) diff --git a/contentcuration/kolibri_public/models.py b/contentcuration/kolibri_public/models.py index 35b3ede984..7b10c4cee1 100644 --- a/contentcuration/kolibri_public/models.py +++ b/contentcuration/kolibri_public/models.py @@ -97,6 +97,8 @@ class AssessmentMetaData(base_models.AssessmentMetaData): class ChannelMetadata(base_models.ChannelMetadata): + # Note: The `categories` field should contain a _list_, NOT a _dict_. + # precalculated fields during annotation/migration published_size = models.BigIntegerField(default=0, null=True, blank=True) total_resource_count = models.IntegerField(default=0, null=True, blank=True) diff --git a/contentcuration/kolibri_public/tests/test_export_channels_to_kolibri_public.py b/contentcuration/kolibri_public/tests/test_export_channels_to_kolibri_public.py index d0e22d4e2c..d7be71395e 100644 --- a/contentcuration/kolibri_public/tests/test_export_channels_to_kolibri_public.py +++ b/contentcuration/kolibri_public/tests/test_export_channels_to_kolibri_public.py @@ -102,10 +102,7 @@ def tearDown(self): def test_export_channel_to_kolibri_public__existing_version( self, mock_channel_mapper ): - categories = { - "Category1": True, - "Category2": True, - } + categories = ["Category1", "Category2"] country1 = Country.objects.create(code="C1", name="Country 1") country2 = Country.objects.create(code="C2", name="Country 2") countries = [country1, country2] diff --git a/contentcuration/kolibri_public/tests/test_mapper.py b/contentcuration/kolibri_public/tests/test_mapper.py index edefafcd25..500f018c33 100644 --- a/contentcuration/kolibri_public/tests/test_mapper.py +++ b/contentcuration/kolibri_public/tests/test_mapper.py @@ -181,7 +181,7 @@ def test_categories__none_provided(self): mapper = ChannelMapper(self.channel) mapper.run() - self.assertEqual(mapper.mapped_channel.categories, {}) + self.assertEqual(mapper.mapped_channel.categories, []) def test_categories__only_provided(self): with using_content_database(self.tempdb): @@ -189,7 +189,7 @@ def test_categories__only_provided(self): channel_id=self.channel.id, ).update(categories=None) - categories = {"Category1": True, "Category2": True} + categories = ["Category1", "Category2"] mapper = ChannelMapper(self.channel, categories=categories) mapper.run() @@ -216,7 +216,7 @@ def test_categories__only_on_content_nodes(self): self.assertEqual( mapper.mapped_channel.categories, - {"Category1": True, "Category2": True, "Category3": True}, + ["Category1", "Category2", "Category3"], ) def test_categories__both_provided_and_on_content_nodes(self): @@ -235,18 +235,13 @@ def test_categories__both_provided_and_on_content_nodes(self): contentnode1.save() contentnode2.save() - categories = {"Category3": True, "Category4": True} + categories = ["Category3", "Category4"] mapper = ChannelMapper(self.channel, categories=categories) mapper.run() self.assertEqual( mapper.mapped_channel.categories, - { - "Category1": True, - "Category2": True, - "Category3": True, - "Category4": True, - }, + ["Category1", "Category2", "Category3", "Category4"], ) def test_countries__none_provided(self): diff --git a/contentcuration/kolibri_public/utils/annotation.py b/contentcuration/kolibri_public/utils/annotation.py index 1c9a193511..05b97f6c44 100644 --- a/contentcuration/kolibri_public/utils/annotation.py +++ b/contentcuration/kolibri_public/utils/annotation.py @@ -21,6 +21,8 @@ def set_channel_metadata_fields( categories=None, countries=None, ): + # Note: The `categories` argument should be a _list_, NOT a _dict_. + # Remove unneeded db_lock channel = ChannelMetadata.objects.get(id=channel_id) calculate_published_size(channel) @@ -92,9 +94,9 @@ def calculate_included_categories(channel, categories): ) ) ) - categories_dict = {category: True for category in contentnode_categories} - categories_dict.update(categories or {}) - channel.categories = categories_dict + + all_categories = sorted(set(categories or []).union(contentnode_categories)) + channel.categories = all_categories channel.save() diff --git a/contentcuration/kolibri_public/utils/export_channel_to_kolibri_public.py b/contentcuration/kolibri_public/utils/export_channel_to_kolibri_public.py index d751bd6614..ddbc19ca7b 100644 --- a/contentcuration/kolibri_public/utils/export_channel_to_kolibri_public.py +++ b/contentcuration/kolibri_public/utils/export_channel_to_kolibri_public.py @@ -23,6 +23,7 @@ def export_channel_to_kolibri_public( categories=None, countries=None, ): + # Note: The `categories` argument should be a _list_, NOT a _dict_. logger.info("Putting channel {} into kolibri_public".format(channel_id)) if channel_version is not None: diff --git a/contentcuration/kolibri_public/utils/mapper.py b/contentcuration/kolibri_public/utils/mapper.py index bd779ea6d2..3f5efdaeb9 100644 --- a/contentcuration/kolibri_public/utils/mapper.py +++ b/contentcuration/kolibri_public/utils/mapper.py @@ -29,6 +29,9 @@ def __init__( # Note: The argument `channel` is an instance of `kolibri_content.models.ChannelMetadata,` # which belongs to a specific channel version to be exported. Therefore, we do not # need to explicitly pass the channel version as an argument here. + + # Note: The `categories` argument should be a _list_, NOT a _dict_. + self.channel = channel self.public = public self.categories = categories From f5b2f751ddad1ffe13071ed99bf2e1666bba172a Mon Sep 17 00:00:00 2001 From: Yeshwanth munagapati Date: Tue, 5 Aug 2025 22:29:20 +0530 Subject: [PATCH 048/472] Removed vuetify components in Invitations in Channels --- .../views/Channel/ChannelInvitation.vue | 158 ++++++++++-------- .../__tests__/channelInvitation.spec.js | 1 + .../channelList/views/ChannelListIndex.vue | 32 ++-- .../frontend/shared/views/StudioRaisedBox.vue | 45 +++++ .../views/__tests__/StudioRaisedBox.spec.js | 20 +++ 5 files changed, 177 insertions(+), 79 deletions(-) create mode 100644 contentcuration/contentcuration/frontend/shared/views/StudioRaisedBox.vue create mode 100644 contentcuration/contentcuration/frontend/shared/views/__tests__/StudioRaisedBox.spec.js diff --git a/contentcuration/contentcuration/frontend/channelList/views/Channel/ChannelInvitation.vue b/contentcuration/contentcuration/frontend/channelList/views/Channel/ChannelInvitation.vue index 8fef4ecee4..ac5d2d3bed 100644 --- a/contentcuration/contentcuration/frontend/channelList/views/Channel/ChannelInvitation.vue +++ b/contentcuration/contentcuration/frontend/channelList/views/Channel/ChannelInvitation.vue @@ -1,75 +1,65 @@