From 889ae6173b53776bfe32fbe45d37bc8faf2c7ed6 Mon Sep 17 00:00:00 2001 From: emjay0921 Date: Wed, 18 Mar 2026 11:49:31 +0800 Subject: [PATCH 1/5] fix(spp_programs): fix deduplication and enrollment interaction bugs --- .../models/managers/deduplication_manager.py | 164 +++++++++++++----- .../models/managers/program_manager.py | 9 +- spp_programs/models/program_membership.py | 52 ++++-- spp_programs/models/programs.py | 32 +++- .../managers/deduplication_manager_view.xml | 3 +- .../views/program_membership_view.xml | 4 + 6 files changed, 202 insertions(+), 62 deletions(-) diff --git a/spp_programs/models/managers/deduplication_manager.py b/spp_programs/models/managers/deduplication_manager.py index cc9cb561..6ad1db1f 100644 --- a/spp_programs/models/managers/deduplication_manager.py +++ b/spp_programs/models/managers/deduplication_manager.py @@ -131,12 +131,18 @@ def _check_duplicate_by_individual_ids(self, beneficiaries): group_of_duplicates[group_membership.individual.id].append(group_membership.group.id) _logger.debug("Found %s duplicate group(s)", len(group_of_duplicates)) - for _individual, group_ids in group_of_duplicates.items(): + for individual_id, group_ids in group_of_duplicates.items(): group_ids_set = set(group_ids) duplicate_beneficiaries = beneficiaries.filtered(lambda rec, gids=group_ids_set: rec.partner_id.id in gids) duplicate_beneficiariy_ids = duplicate_beneficiaries.mapped("id") - self._record_duplicate(self, duplicate_beneficiariy_ids, "Duplicate individuals") + individual_rec = self.env["res.partner"].browse(individual_id) + group_names = duplicate_beneficiaries.mapped("partner_id.name") + reason = ( + f"Shared member: {individual_rec.name} " + f"found in {len(group_ids)} groups ({', '.join(group_names)})" + ) + self._record_duplicate(self, duplicate_beneficiariy_ids, reason) duplicated_enrolled = duplicate_beneficiaries.filtered(lambda rec: rec.state == "enrolled") if len(duplicated_enrolled) == 1: @@ -158,7 +164,11 @@ class IDDocumentDeduplication(models.Model): _inherit = ["spp.base.deduplication.manager", "spp.manager.source.mixin"] _description = "ID Deduplication Manager" - supported_id_document_type_ids = fields.Many2many("spp.id.type", string="Supported ID Document Types") + supported_id_document_type_ids = fields.Many2many( + "spp.vocabulary.code", + string="Supported ID Document Types", + domain=[("vocabulary_id.namespace_uri", "=", "urn:openspp:vocab:id-type")], + ) def deduplicate_beneficiaries(self, states): for rec in self: @@ -233,7 +243,7 @@ def _check_duplicate_by_group_with_individual(self, beneficiaries): if x.id_type_id in self.supported_id_document_type_ids and ( (not x.expiry_date) or x.expiry_date > date.today() ): - id_doc_id_with_id_type_and_value = {x.id: x.id_type_id.name + "-" + x.value} + id_doc_id_with_id_type_and_value = {x.id: x.id_type_id.display + "-" + x.value} individual_id_docs.update(id_doc_id_with_id_type_and_value) # Check ID Docs of each group @@ -242,7 +252,7 @@ def _check_duplicate_by_group_with_individual(self, beneficiaries): if x.id_type_id in self.supported_id_document_type_ids and ( (not x.expiry_date) or x.expiry_date > date.today() ): - id_doc_id_with_id_type_and_value = {x.id: x.id_type_id.name + "-" + x.value} + id_doc_id_with_id_type_and_value = {x.id: x.id_type_id.display + "-" + x.value} individual_id_docs.update(id_doc_id_with_id_type_and_value) _logger.debug("Found %s ID document(s) to check", len(individual_id_docs)) @@ -264,6 +274,13 @@ def _check_duplicate_by_group_with_individual(self, beneficiaries): ) _logger.debug("Found %s group(s) with duplicates", len(group_with_duplicates)) + # Build mapping: individual_id -> list of duplicate ID doc descriptions + individual_dup_docs = {} + for doc in duplicated_doc_ids: + individual_dup_docs.setdefault(doc.partner_id.id, []).append( + f"{doc.id_type_id.display}: {doc.value}" + ) + group_of_duplicates = {} for group_membership in group_with_duplicates: _logger.debug("Processing group membership duplicate") @@ -272,12 +289,18 @@ def _check_duplicate_by_group_with_individual(self, beneficiaries): group_of_duplicates[group_membership.individual.id].append(group_membership.group.id) _logger.debug("Found %s duplicate group(s)", len(group_of_duplicates)) - for _individual, group_ids in group_of_duplicates.items(): + for individual_id, group_ids in group_of_duplicates.items(): group_ids_set = set(group_ids) duplicate_beneficiaries = beneficiaries.filtered(lambda rec, gids=group_ids_set: rec.partner_id.id in gids) duplicate_beneficiariy_ids = duplicate_beneficiaries.mapped("id") - self._record_duplicate(self, duplicate_beneficiariy_ids, "Duplicate ID Documents") + individual_rec = self.env["res.partner"].browse(individual_id) + doc_info = ", ".join(individual_dup_docs.get(individual_id, [])) + reason = ( + f"Duplicate ID document ({doc_info}) " + f"on member: {individual_rec.name}" + ) + self._record_duplicate(self, duplicate_beneficiariy_ids, reason) duplicated_enrolled = duplicate_beneficiaries.filtered(lambda rec: rec.state == "enrolled") if len(duplicated_enrolled) == 1: @@ -295,55 +318,68 @@ def _check_duplicate_by_group_with_individual(self, beneficiaries): def _check_duplicate_by_individual(self, beneficiaries): """ - This method is used to check if there are any duplicates - among the individuals id docs. - :param beneficiary_ids: The beneficiaries. - :return: + Check for duplicate ID documents among individual beneficiaries. + Groups duplicates by shared ID value and applies keep-one-enrolled logic. """ _logger.debug("-" * 100) individual_ids = beneficiaries.mapped("partner_id.id") individuals = self.env["res.partner"].search([("id", "in", individual_ids)]) _logger.debug("Checking ID Document Duplicates for %s individual(s)", len(individuals)) + # Map: reg_id.id -> "IDType-Value", also track reg_id -> partner_id individual_id_docs = {} - # Check ID Documents of each individual + reg_id_to_partner = {} for i in individuals: for x in i.reg_ids: if x.id_type_id in self.supported_id_document_type_ids and ( (not x.expiry_date) or x.expiry_date > date.today() ): - id_doc_id_with_id_type_and_value = {x.id: x.id_type_id.name + "-" + x.value} - individual_id_docs.update(id_doc_id_with_id_type_and_value) + doc_key = x.id_type_id.display + "-" + x.value + individual_id_docs[x.id] = doc_key + reg_id_to_partner[x.id] = i.id _logger.debug("Found %s ID document(s) to check", len(individual_id_docs)) rev_dict = {} for key, value in individual_id_docs.items(): rev_dict.setdefault(value, set()).add(key) - duplicate_ids = filter(lambda x: len(x) > 1, rev_dict.values()) - duplicate_ids = list(duplicate_ids) - duplicate_ids = list(itertools.chain.from_iterable(duplicate_ids)) - _logger.debug("Found %s duplicate ID document(s)", len(duplicate_ids)) + all_duplicated_memberships = self.env["spp.program.membership"] - duplicated_doc_ids = self.env["spp.registry.id"].search([("id", "in", duplicate_ids)]) - individual_ids = [x.partner_id.id for x in duplicated_doc_ids] - individual_ids = list(dict.fromkeys(individual_ids)) - _logger.debug("Found %s individual(s) with duplicated ID documents", len(individual_ids)) - individual_program_membership = self.env["spp.program.membership"].search( - [ - ("partner_id", "in", individual_ids), - ("program_id", "=", self.program_id.id), - ] - ) + # Process each group of duplicates sharing the same ID value + for doc_key, reg_ids in rev_dict.items(): + if len(reg_ids) <= 1: + continue + + partner_ids = list({reg_id_to_partner[rid] for rid in reg_ids}) + dup_memberships = self.env["spp.program.membership"].search( + [ + ("partner_id", "in", partner_ids), + ("program_id", "=", self.program_id.id), + ] + ) + if not dup_memberships: + continue + + names = dup_memberships.mapped("partner_id.name") + reason = ( + f"Duplicate ID document ({doc_key}) " + f"shared by: {', '.join(names)}" + ) + self._record_duplicate(self, dup_memberships.ids, reason) - for duplicates in individual_program_membership: - duplicate_individuals = [duplicates.id] - self._record_duplicate(self, duplicate_individuals, "Duplicate ID Documents") + # Keep-one-enrolled logic + duplicated_enrolled = dup_memberships.filtered(lambda rec: rec.state == "enrolled") + if len(duplicated_enrolled) == 1: + to_mark = dup_memberships.filtered(lambda rec: rec.state != "enrolled") + else: + to_mark = dup_memberships + to_mark.filtered( + lambda rec: rec.state not in ["exited", "not_eligible", "duplicated"] + ).write({"state": "duplicated"}) - if duplicates.state == "enrolled": - duplicates.write({"state": "duplicated"}) + all_duplicated_memberships |= dup_memberships - return individual_program_membership + return all_duplicated_memberships class PhoneNumberDeduplication(models.Model): @@ -462,6 +498,12 @@ def _check_duplicate_by_group_with_individual(self, beneficiaries): ) _logger.debug("Found %s group(s) with duplicates", len(group_with_duplicates)) + # Build mapping: individual_id -> list of duplicate phone numbers + individual_dup_phones = {} + for phone_rec in duplicate_individuals_ids: + individual_dup_phones.setdefault(phone_rec.partner_id.id, []).append( + phone_rec.phone_no + ) group_of_duplicates = {} for group_membership in group_with_duplicates: @@ -471,12 +513,18 @@ def _check_duplicate_by_group_with_individual(self, beneficiaries): group_of_duplicates[group_membership.individual.id].append(group_membership.group.id) _logger.debug("Found %s duplicate group(s)", len(group_of_duplicates)) - for _individual, group_ids in group_of_duplicates.items(): + for individual_id, group_ids in group_of_duplicates.items(): group_ids_set = set(group_ids) duplicate_beneficiaries = beneficiaries.filtered(lambda rec, gids=group_ids_set: rec.partner_id.id in gids) duplicate_beneficiariy_ids = duplicate_beneficiaries.mapped("id") - self._record_duplicate(self, duplicate_beneficiariy_ids, "Duplicate Phone Numbers") + individual_rec = self.env["res.partner"].browse(individual_id) + phone_info = ", ".join(individual_dup_phones.get(individual_id, [])) + reason = ( + f"Duplicate phone number ({phone_info}) " + f"on member: {individual_rec.name}" + ) + self._record_duplicate(self, duplicate_beneficiariy_ids, reason) duplicated_enrolled = duplicate_beneficiaries.filtered(lambda rec: rec.state == "enrolled") if len(duplicated_enrolled) == 1: @@ -531,14 +579,46 @@ def _check_duplicate_by_individual(self, beneficiaries): ] ) - for duplicates in individual_program_membership: - duplicate_individuals = [duplicates.id] - self._record_duplicate(self, duplicate_individuals, "Duplicate Phone Numbers") + all_duplicated_memberships = self.env["spp.program.membership"] + + # Build reverse map: phone_sanitized -> list of partner_ids + phone_to_partners = {} + for phone_rec in duplicated_phone_ids: + phone_to_partners.setdefault(phone_rec.phone_sanitized, set()).add(phone_rec.partner_id.id) + + for phone_val, partner_id_set in phone_to_partners.items(): + if len(partner_id_set) <= 1: + continue + + dup_memberships = self.env["spp.program.membership"].search( + [ + ("partner_id", "in", list(partner_id_set)), + ("program_id", "=", self.program_id.id), + ] + ) + if not dup_memberships: + continue + + names = dup_memberships.mapped("partner_id.name") + reason = ( + f"Duplicate phone number ({phone_val}) " + f"shared by: {', '.join(names)}" + ) + self._record_duplicate(self, dup_memberships.ids, reason) + + # Keep-one-enrolled logic + duplicated_enrolled = dup_memberships.filtered(lambda rec: rec.state == "enrolled") + if len(duplicated_enrolled) == 1: + to_mark = dup_memberships.filtered(lambda rec: rec.state != "enrolled") + else: + to_mark = dup_memberships + to_mark.filtered( + lambda rec: rec.state not in ["exited", "not_eligible", "duplicated"] + ).write({"state": "duplicated"}) - if duplicates.state == "enrolled": - duplicates.write({"state": "duplicated"}) + all_duplicated_memberships |= dup_memberships - return individual_program_membership + return all_duplicated_memberships class IDPhoneEligibilityManager(models.Model): diff --git a/spp_programs/models/managers/program_manager.py b/spp_programs/models/managers/program_manager.py index 125331c2..0a2598b2 100644 --- a/spp_programs/models/managers/program_manager.py +++ b/spp_programs/models/managers/program_manager.py @@ -214,8 +214,12 @@ def _enroll_eligible_registrants(self, states, offset=0, limit=None, do_count=Fa for el in eligibility_managers: members = el.enroll_eligible_registrants(members) # enroll the one not already enrolled: + # Exclude members that are duplicated or exited — those states + # should only be changed through their own workflows. _logger.debug("members filtered: %s", members) - not_enrolled = members.filtered(lambda m: m.state != "enrolled") + not_enrolled = members.filtered( + lambda m: m.state not in ("enrolled", "duplicated", "exited") + ) _logger.debug("not_enrolled: %s", not_enrolled) not_enrolled.write( { @@ -226,7 +230,8 @@ def _enroll_eligible_registrants(self, states, offset=0, limit=None, do_count=Fa # dis-enroll the one not eligible anymore: enrolled_members_ids = members.ids members_to_remove = member_before.filtered( - lambda m: m.state != "not_eligible" and m.id not in enrolled_members_ids + lambda m: m.state not in ("not_eligible", "duplicated", "exited") + and m.id not in enrolled_members_ids ) # _logger.debug("members_to_remove: %s", members_to_remove) members_to_remove.write( diff --git a/spp_programs/models/program_membership.py b/spp_programs/models/program_membership.py index ee4132ae..c5ee399c 100644 --- a/spp_programs/models/program_membership.py +++ b/spp_programs/models/program_membership.py @@ -52,6 +52,23 @@ class SPPProgramMembership(models.Model): registrant_id = fields.Integer(string="Registrant ID", related="partner_id.id") + duplicate_reason = fields.Char( + string="Duplicate Reason", + compute="_compute_duplicate_reason", + ) + + def _compute_duplicate_reason(self): + for rec in self: + if rec.state == "duplicated": + dup_records = self.env["spp.program.membership.duplicate"].search( + [("beneficiary_ids", "in", rec.id), ("state", "=", "duplicate")], + order="id desc", + limit=1, + ) + rec.duplicate_reason = dup_records.reason if dup_records else False + else: + rec.duplicate_reason = False + @api.constrains("partner_id", "program_id") def _check_unique_partner_per_program(self): # Prefetch partner_id and program_id to avoid N+1 queries in loop @@ -212,7 +229,13 @@ def enroll_eligible_registrants(self): member = em.enroll_eligible_registrants(member) if len(member) > 0: - if self.state != "enrolled": + if self.state in ("duplicated", "exited"): + message = _( + "Cannot enroll: beneficiary is currently %s.", + dict(self._fields["state"].selection).get(self.state, self.state), + ) + kind = "warning" + elif self.state != "enrolled": self.write( { "state": "enrolled", @@ -221,19 +244,22 @@ def enroll_eligible_registrants(self): ) message = _("%s Beneficiaries enrolled.", len(member)) kind = "success" - return { - "type": "ir.actions.client", - "tag": "display_notification", - "params": { - "title": _("Enrollment"), - "message": message, - "sticky": False, - "type": kind, - "next": { - "type": "ir.actions.act_window_close", - }, + else: + message = _("Beneficiary is already enrolled.") + kind = "info" + return { + "type": "ir.actions.client", + "tag": "display_notification", + "params": { + "title": _("Enrollment"), + "message": message, + "sticky": False, + "type": kind, + "next": { + "type": "ir.actions.act_window_close", }, - } + }, + } else: self.state = "not_eligible" diff --git a/spp_programs/models/programs.py b/spp_programs/models/programs.py index a1ad8016..01fc0332 100644 --- a/spp_programs/models/programs.py +++ b/spp_programs/models/programs.py @@ -380,17 +380,42 @@ def deduplicate_beneficiaries(self): message = None kind = "success" if len(deduplication_managers): + # Count already-flagged duplicates before running + already_duplicated = self.env["spp.program.membership"].search_count( + [("program_id", "=", rec.id), ("state", "=", "duplicated")] + ) + states = ["draft", "enrolled", "eligible", "paused", "duplicated"] duplicates = 0 for el in deduplication_managers: duplicates += el.deduplicate_beneficiaries(states) - if duplicates > 0: - message = _("%s Instances of Beneficiaries duplicate.", duplicates) + # Count total duplicates after running + total_duplicated = self.env["spp.program.membership"].search_count( + [("program_id", "=", rec.id), ("state", "=", "duplicated")] + ) + new_duplicates = total_duplicated - already_duplicated + + if total_duplicated > 0: + parts = [] + if new_duplicates > 0: + parts.append( + _("%(new)s new duplicate(s) found", new=new_duplicates) + ) + if already_duplicated > 0: + parts.append( + _("%(existing)s already flagged", existing=already_duplicated) + ) + message = ", ".join(parts) + "." + kind = "warning" + elif duplicates > 0: + message = _( + "Found %(count)s duplicate beneficiaries.", + count=duplicates, + ) kind = "warning" else: message = _("No duplicates found.") - kind = "success" else: raise UserError(_("No Deduplication Manager defined.")) @@ -399,7 +424,6 @@ def deduplicate_beneficiaries(self): "type": "ir.actions.client", "tag": "display_notification", "params": { - "title": _("Deduplication"), "message": message, "sticky": False, "type": kind, diff --git a/spp_programs/views/managers/deduplication_manager_view.xml b/spp_programs/views/managers/deduplication_manager_view.xml index 0e92cf6b..d1220a77 100644 --- a/spp_programs/views/managers/deduplication_manager_view.xml +++ b/spp_programs/views/managers/deduplication_manager_view.xml @@ -172,7 +172,8 @@ Part of OpenSPP. See LICENSE file for full copyright and licensing details. diff --git a/spp_programs/views/program_membership_view.xml b/spp_programs/views/program_membership_view.xml index f01e3670..2fdf13c8 100644 --- a/spp_programs/views/program_membership_view.xml +++ b/spp_programs/views/program_membership_view.xml @@ -122,6 +122,10 @@ Part of OpenSPP. See LICENSE file for full copyright and licensing details. bg_color="bg-danger" invisible="state != 'not_eligible'" /> +