From faec3888983fd83efc9ef37335026b295dec6445 Mon Sep 17 00:00:00 2001 From: Mithun James <1007084+drtechie@users.noreply.github.com> Date: Thu, 29 Oct 2026 11:22:29 +0530 Subject: [PATCH 01/12] feat(stoptb): /stoptb/sync/changes feed on the hub change log One endpoint replaces the 15+ getAll calls a STOP-TB pull makes. With no token it pages through everything in the requested villages (keyset, type by type, beneficiaries first); after that it returns only records with t_sync_change_log lines after the tablet's token, in their current state, deduplicated, with deletes. Tokens carry the hub's server_uuid, so a token from another hub or older than the log's retention starts a fresh snapshot. Records keep the exact JSON each existing endpoint returns, via by-id variants of the existing builders (beneficiary, screening, examination, OPD, diagnostics, suspected, confirmed, form responses, diagnostic orders), so the app reuses its parsers. The feed is served only where the change log exists and every source table has its triggers (checked on the hub); elsewhere it answers {supported:false} and the app keeps the full pull. StopTBSyncCoverageTest fails when a mapped table is neither synced nor listed as deliberately not synced. Co-Authored-By: Claude Opus 5.5 --- .../flw/controller/StopTBSyncController.java | 59 +++ .../iemr/flw/service/BeneficiaryService.java | 3 + .../flw/service/DiagnosticOrderService.java | 8 + .../service/DynamicFormResponseService.java | 6 + .../service/StopTBPrescriptionService.java | 39 +- .../com/iemr/flw/service/StopTBService.java | 12 + .../iemr/flw/service/StopTBSyncService.java | 15 + .../flw/service/TBConfirmedCaseService.java | 3 + .../iemr/flw/service/TBSuspectedService.java | 3 + .../service/impl/BeneficiaryServiceImpl.java | 41 ++- .../impl/DiagnosticOrderServiceImpl.java | 35 ++ .../impl/DynamicFormResponseServiceImpl.java | 20 + .../flw/service/impl/StopTBServiceImpl.java | 55 +++ .../service/impl/StopTBSyncServiceImpl.java | 342 ++++++++++++++++++ .../impl/TBConfirmedCaseServiceImpl.java | 31 +- .../service/impl/TBSuspectedServiceImpl.java | 22 ++ .../iemr/flw/service/sync/StopTBSyncType.java | 128 +++++++ .../com/iemr/flw/service/sync/SyncToken.java | 97 +++++ .../service/sync/StopTBSyncCoverageTest.java | 107 ++++++ .../iemr/flw/service/sync/SyncTokenTest.java | 46 +++ 20 files changed, 1052 insertions(+), 20 deletions(-) create mode 100644 src/main/java/com/iemr/flw/controller/StopTBSyncController.java create mode 100644 src/main/java/com/iemr/flw/service/StopTBSyncService.java create mode 100644 src/main/java/com/iemr/flw/service/impl/StopTBSyncServiceImpl.java create mode 100644 src/main/java/com/iemr/flw/service/sync/StopTBSyncType.java create mode 100644 src/main/java/com/iemr/flw/service/sync/SyncToken.java create mode 100644 src/test/java/com/iemr/flw/service/sync/StopTBSyncCoverageTest.java create mode 100644 src/test/java/com/iemr/flw/service/sync/SyncTokenTest.java diff --git a/src/main/java/com/iemr/flw/controller/StopTBSyncController.java b/src/main/java/com/iemr/flw/controller/StopTBSyncController.java new file mode 100644 index 000000000..9c1429769 --- /dev/null +++ b/src/main/java/com/iemr/flw/controller/StopTBSyncController.java @@ -0,0 +1,59 @@ +package com.iemr.flw.controller; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.iemr.flw.service.StopTBSyncService; +import com.iemr.flw.utils.response.OutputResponse; +import io.swagger.v3.oas.annotations.Operation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.*; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +@RestController +@RequestMapping("/stoptb/sync") +public class StopTBSyncController { + + private final Logger logger = LoggerFactory.getLogger(StopTBSyncController.class); + + private final Gson gson = new GsonBuilder().serializeNulls().create(); + + @Autowired + private StopTBSyncService stopTBSyncService; + + /** + * Body: { token: string|null, providerServiceMapID: int, villageIds: [int], limit: int }. + * Response: { supported, serverId, snapshot, items: [{type, op, key, version, data}], nextToken, hasMore }. + * Call again with nextToken while hasMore is true. On a server without the hub change log the + * response is { supported: false } and the app keeps using the getAll endpoints. + */ + @PostMapping("/changes") + @Operation(summary = "STOP-TB records changed since the token (first call: everything in the villages)") + public String getChanges(@RequestBody Map body, + @RequestHeader(value = "Authorization", required = false) String authorization) { + OutputResponse response = new OutputResponse(); + try { + Object token = body.get("token"); + Integer psm = body.get("providerServiceMapID") != null + ? Integer.parseInt(body.get("providerServiceMapID").toString()) : null; + List villageIds = new ArrayList<>(); + if (body.get("villageIds") instanceof List) { + for (Object v : (List) body.get("villageIds")) { + if (v != null) villageIds.add((int) Double.parseDouble(v.toString())); + } + } + Integer limit = body.get("limit") != null ? (int) Double.parseDouble(body.get("limit").toString()) : null; + Map data = stopTBSyncService.getChanges( + token != null ? token.toString() : null, psm, villageIds, limit, authorization); + response.setResponse(gson.toJson(data)); + } catch (Exception e) { + logger.error("Error in sync/changes: " + e); + response.setError(5000, "Error fetching changes: " + e.getMessage()); + } + return response.toString(); + } +} diff --git a/src/main/java/com/iemr/flw/service/BeneficiaryService.java b/src/main/java/com/iemr/flw/service/BeneficiaryService.java index e69bc887f..d043e7eea 100644 --- a/src/main/java/com/iemr/flw/service/BeneficiaryService.java +++ b/src/main/java/com/iemr/flw/service/BeneficiaryService.java @@ -5,11 +5,14 @@ import com.iemr.flw.dto.iemr.SAMResponseDTO; import java.util.List; +import java.util.Map; public interface BeneficiaryService { String getBenData(GetBenRequestHandler requestDTO, String authorisation) throws Exception; + Map getBeneficiaryRecordsByRegIds(List benRegIds, String authorisation); + String saveEyeCheckupVsit(List eyeCheckupRequestDTOS,String token); List getEyeCheckUpVisit(GetBenRequestHandler request,String token); diff --git a/src/main/java/com/iemr/flw/service/DiagnosticOrderService.java b/src/main/java/com/iemr/flw/service/DiagnosticOrderService.java index f2f39dba6..42fb1c337 100644 --- a/src/main/java/com/iemr/flw/service/DiagnosticOrderService.java +++ b/src/main/java/com/iemr/flw/service/DiagnosticOrderService.java @@ -48,6 +48,14 @@ public interface DiagnosticOrderService { DiagnosticOrderResultDto getOrderResult(Long beneficiaryId, String orderType, Long visitCode); + /** + * Orders with their result, keyed by order id (STOP-TB changes feed). Deleted orders are left + * out. Each record carries what getBeneficiariesByStatus and getOrderResult give today: the + * beneficiary, order type, visit code and status (the app buckets by the latest order per + * beneficiary and type), plus the result fields. + */ + java.util.Map> getOrderRecordsByIds(java.util.Collection ids); + DiagnosticOrder getOrder(Long beneficiaryId, String orderType, Long visitCode) throws Exception; List getOrdersByBeneficiaryId(Long beneficiaryId) throws Exception; diff --git a/src/main/java/com/iemr/flw/service/DynamicFormResponseService.java b/src/main/java/com/iemr/flw/service/DynamicFormResponseService.java index 1f2255f3e..8265cfafb 100644 --- a/src/main/java/com/iemr/flw/service/DynamicFormResponseService.java +++ b/src/main/java/com/iemr/flw/service/DynamicFormResponseService.java @@ -55,6 +55,12 @@ List getResponsesByBeneficiary(Long beneficiaryId, Integer vill /** Single response with all nested section and question answers. */ FormResponseDTO getResponseById(Long responseId); + /** + * Responses with all nested answers, keyed by responseId, as getResponsesByBeneficiary returns + * them (STOP-TB changes feed). Ids with no response are left out. + */ + java.util.Map getResponsesByIds(java.util.Collection responseIds); + /** * Submit multiple form responses in one bulk transaction. * All FormResponse rows are batch-inserted via saveAll() so responseIds are available diff --git a/src/main/java/com/iemr/flw/service/StopTBPrescriptionService.java b/src/main/java/com/iemr/flw/service/StopTBPrescriptionService.java index b846bc022..740057107 100644 --- a/src/main/java/com/iemr/flw/service/StopTBPrescriptionService.java +++ b/src/main/java/com/iemr/flw/service/StopTBPrescriptionService.java @@ -320,16 +320,43 @@ public List> getStandardOpdRecords(Integer providerServiceMa ? "AND v.BeneficiaryRegID IN (SELECT f.beneficiary_reg_id FROM db_iemr.i_ben_flow_outreach f " + "WHERE f.providerServiceMapID = :psm AND f.villageID = :villageID) " : ""; + Map params = new HashMap<>(); + params.put("psm", providerServiceMapID); + if (villageID != null) params.put("villageID", villageID); + return standardOpdRecords("v.ProviderServiceMapID = :psm " + villageFilter, params); + } + + // The same records for specific visits (BenVisitID), for the STOP-TB changes feed. A visit that is + // deleted or has no complaint or drug is left out, exactly as getStandardOpdRecords leaves it out. + public List> getStandardOpdRecordsByVisitIds(Collection benVisitIds) { + if (benVisitIds.isEmpty()) return new ArrayList<>(); + Map params = new HashMap<>(); + params.put("ids", benVisitIds); + return standardOpdRecords("v.BenVisitID IN (:ids) ", params); + } + + // Visit codes among these that getStandardOpdRecords would return (not deleted, with a complaint or drug). + public Set visitCodesWithStandardOpd(Collection visitCodes) { + Set out = new HashSet<>(); + if (visitCodes.isEmpty()) return out; + Map params = new HashMap<>(); + params.put("codes", visitCodes); + for (Map m : standardOpdRecords("v.VisitCode IN (:codes) ", params)) { + out.add((Long) m.get("visitCode")); + } + return out; + } + + private List> standardOpdRecords(String visitFilter, Map params) { Query visitQuery = em.createNativeQuery("SELECT v.BenVisitID, v.BeneficiaryRegID, v.VisitCode, v.CreatedBy, " - + "v.CreatedDate, v.LastModDate FROM db_iemr.t_benvisitdetail v " - + "WHERE v.ProviderServiceMapID = :psm AND v.Deleted = false " + + "v.CreatedDate, v.LastModDate, v.ProviderServiceMapID FROM db_iemr.t_benvisitdetail v " + + "WHERE " + visitFilter + "AND v.Deleted = false " + "AND (EXISTS (SELECT 1 FROM db_iemr.t_benchiefcomplaint c WHERE c.BeneficiaryRegID = v.BeneficiaryRegID " + " AND c.VisitCode = v.VisitCode AND c.ChiefComplaintID IS NOT NULL AND c.Deleted = false) " + " OR EXISTS (SELECT 1 FROM db_iemr.t_prescribeddrug d WHERE d.BeneficiaryRegID = v.BeneficiaryRegID " + " AND d.VisitCode = v.VisitCode AND d.DrugID IS NOT NULL AND d.Deleted = false)) " - + villageFilter + "ORDER BY v.CreatedDate DESC") - .setParameter("psm", providerServiceMapID); - if (villageID != null) visitQuery.setParameter("villageID", villageID); + + "ORDER BY v.CreatedDate DESC"); + params.forEach(visitQuery::setParameter); List visits = visitQuery.getResultList(); Map> byVisit = new LinkedHashMap<>(); @@ -338,7 +365,7 @@ public List> getStandardOpdRecords(Integer providerServiceMa m.put("id", ((Number) v[0]).longValue()); m.put("beneficiaryRegID", ((Number) v[1]).longValue()); m.put("visitCode", ((Number) v[2]).longValue()); - m.put("providerServiceMapID", providerServiceMapID); + m.put("providerServiceMapID", v[6] != null ? ((Number) v[6]).intValue() : null); m.put("chiefComplaint", new ArrayList()); m.put("medication", new StringJoiner(", ")); m.put("dosage", new StringJoiner(", ")); diff --git a/src/main/java/com/iemr/flw/service/StopTBService.java b/src/main/java/com/iemr/flw/service/StopTBService.java index e549daa6f..4011d4315 100644 --- a/src/main/java/com/iemr/flw/service/StopTBService.java +++ b/src/main/java/com/iemr/flw/service/StopTBService.java @@ -1,5 +1,6 @@ package com.iemr.flw.service; +import java.util.Collection; import java.util.List; import java.util.Map; @@ -19,4 +20,15 @@ public interface StopTBService { // Nurse — Diagnostics List> saveDiagnostics(List> dataList) throws Exception; Map getAllDiagnostics(Integer providerServiceMapID, Integer villageID) throws Exception; + + // The same records the getAll endpoints return, for specific rows (STOP-TB changes feed), keyed by + // the row id. Deleted rows, and rows getAll would not return, are left out. + Map> getNurseTBScreeningsByIds(Collection ids); + Map> getGeneralExaminationsByIds(Collection ids); + Map> getDiagnosticsByIds(Collection ids); + // Standard-table OPD visits, keyed by BenVisitID + Map> getGeneralOpdByVisitIds(Collection benVisitIds); + // Older free-text OPD rows (tb_stoptb_general_opd), keyed by their id; left out when the same visit + // also exists in the standard tables, as getAllGeneralOpd does + Map> getLegacyGeneralOpdByIds(Collection ids); } diff --git a/src/main/java/com/iemr/flw/service/StopTBSyncService.java b/src/main/java/com/iemr/flw/service/StopTBSyncService.java new file mode 100644 index 000000000..81fa47c17 --- /dev/null +++ b/src/main/java/com/iemr/flw/service/StopTBSyncService.java @@ -0,0 +1,15 @@ +package com.iemr.flw.service; + +import java.util.List; +import java.util.Map; + +public interface StopTBSyncService { + + /** + * One page of the STOP-TB changes feed for the given villages: a first full download when the + * token is missing, from another hub or too old; otherwise only what changed after it. + * Returns {supported:false} where the hub change log isn't installed, so the app keeps its full pull. + */ + Map getChanges(String token, Integer providerServiceMapID, List villageIds, + Integer limit, String authorisation); +} diff --git a/src/main/java/com/iemr/flw/service/TBConfirmedCaseService.java b/src/main/java/com/iemr/flw/service/TBConfirmedCaseService.java index 6471f7996..b1d4165eb 100644 --- a/src/main/java/com/iemr/flw/service/TBConfirmedCaseService.java +++ b/src/main/java/com/iemr/flw/service/TBConfirmedCaseService.java @@ -15,4 +15,7 @@ public interface TBConfirmedCaseService { String getByUserId(String authorisation) throws Exception; String getByProviderServiceMapId(Integer providerServiceMapID, Integer villageID) throws Exception; + + // One record per row, in the same JSON shape getAll returns (STOP-TB changes feed), keyed by id. + java.util.Map getRecordsByIds(java.util.Collection ids); } diff --git a/src/main/java/com/iemr/flw/service/TBSuspectedService.java b/src/main/java/com/iemr/flw/service/TBSuspectedService.java index 32d882a80..97d9e0d87 100644 --- a/src/main/java/com/iemr/flw/service/TBSuspectedService.java +++ b/src/main/java/com/iemr/flw/service/TBSuspectedService.java @@ -13,4 +13,7 @@ public interface TBSuspectedService { String save(TBSuspectedRequestDTO tbScreeningList) throws Exception; String getByUserId(GetBenRequestHandler userId); + + // One record per row, in the same JSON shape getAll returns (STOP-TB changes feed), keyed by id. + java.util.Map getRecordsByIds(java.util.Collection ids); } diff --git a/src/main/java/com/iemr/flw/service/impl/BeneficiaryServiceImpl.java b/src/main/java/com/iemr/flw/service/impl/BeneficiaryServiceImpl.java index 7136f4905..8bda704c8 100644 --- a/src/main/java/com/iemr/flw/service/impl/BeneficiaryServiceImpl.java +++ b/src/main/java/com/iemr/flw/service/impl/BeneficiaryServiceImpl.java @@ -12,6 +12,7 @@ import java.util.ArrayList; import java.util.Calendar; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -220,6 +221,38 @@ public String getBenData(GetBenRequestHandler request, String authorisation) thr private String getMappingsForAddressIDs(List addressList, int totalPage, String authorisation) { + List> resultList = buildBeneficiaryRecords(addressList, authorisation); + + Map response = new HashMap<>(); + response.put("data", resultList); + response.put("pageSize", Integer.parseInt(door_to_door_page_size)); + response.put("totalPage", totalPage); + Gson gson = new GsonBuilder().setDateFormat("MMM dd, yyyy h:mm:ss a").create(); + return gson.toJson(response); + } + + // One record per beneficiary, in the same JSON shape getBeneficiaryData returns, keyed by BenRegId. + // Used by the STOP-TB changes feed; a BenRegId with no mapping or address is left out. + @Override + public Map getBeneficiaryRecordsByRegIds(List benRegIds, String authorisation) { + List addresses = new ArrayList<>(); + for (Long benRegId : benRegIds) { + RMNCHMBeneficiarymapping m = beneficiaryRepo.getById(BigInteger.valueOf(benRegId)); + if (m == null || m.getBenAddressId() == null) continue; + RMNCHMBeneficiaryaddress a = beneficiaryRepo.getAddressById(m.getBenAddressId()); + if (a != null) addresses.add(a); + } + Gson gson = new GsonBuilder().setDateFormat("MMM dd, yyyy h:mm:ss a").create(); + Map records = new LinkedHashMap<>(); + for (Map record : buildBeneficiaryRecords(addresses, authorisation)) { + Object benRegId = record.get("BenRegId"); + if (benRegId != null) records.put(new BigInteger(benRegId.toString()).longValue(), gson.toJson(record)); + } + return records; + } + + private List> buildBeneficiaryRecords(List addressList, + String authorisation) { RMNCHHouseHoldDetails benHouseHoldRMNCH_ROBJ; RMNCHBeneficiaryDetailsRmnch benDetailsRMNCH_OBJ; RMNCHBornBirthDetails benBotnBirthRMNCH_ROBJ; @@ -616,13 +649,7 @@ private String getMappingsForAddressIDs(List addressLi logger.error("error for addressID :" + e.getMessage() + a.getId() + " and vanID : " + a.getVanID()); } } - - Map response = new HashMap<>(); - response.put("data", resultList); - response.put("pageSize", Integer.parseInt(door_to_door_page_size)); - response.put("totalPage", totalPage); - Gson gson = new GsonBuilder().setDateFormat("MMM dd, yyyy h:mm:ss a").create(); - return gson.toJson(response); + return resultList; } diff --git a/src/main/java/com/iemr/flw/service/impl/DiagnosticOrderServiceImpl.java b/src/main/java/com/iemr/flw/service/impl/DiagnosticOrderServiceImpl.java index 541210707..19a00862d 100644 --- a/src/main/java/com/iemr/flw/service/impl/DiagnosticOrderServiceImpl.java +++ b/src/main/java/com/iemr/flw/service/impl/DiagnosticOrderServiceImpl.java @@ -765,6 +765,41 @@ public DiagnosticOrderResultDto getOrderResult(Long beneficiaryId, String orderT return dto; } + @Override + public java.util.Map> getOrderRecordsByIds(java.util.Collection ids) { + java.util.Map> out = new java.util.LinkedHashMap<>(); + for (DiagnosticOrder order : diagnosticOrderRepo.findAllById(ids)) { + if (Boolean.TRUE.equals(order.getDeleted())) continue; + DiagnosticOrderResultDto dto = toResultDto(order); + if (order.getExternalOrderId() != null) { + diagnosticResultRepo.findByExternalOrderIdAndDeletedFalse(order.getExternalOrderId()).ifPresent(result -> { + dto.setProviderStatus(result.getProviderStatus()); + dto.setResultSummary(result.getResultSummary()); + dto.setTbPresence(result.getTbPresence()); + dto.setTbConfidence(result.getTbConfidence()); + dto.setDrugResistancePresence(result.getDrugResistancePresence()); + }); + } + java.util.Map m = new java.util.LinkedHashMap<>(); + m.put("id", order.getId()); + m.put("beneficiaryId", order.getBeneficiaryId()); + m.put("visitCode", order.getVisitCode()); + m.put("createdDate", order.getCreatedDate()); + m.put("externalOrderId", dto.getExternalOrderId()); + m.put("orderType", dto.getOrderType()); + m.put("status", dto.getStatus()); + m.put("providerStatus", dto.getProviderStatus()); + m.put("resultSummary", dto.getResultSummary()); + m.put("errorMessage", dto.getErrorMessage()); + m.put("reasonToClose", dto.getReasonToClose()); + m.put("tbPresence", dto.getTbPresence()); + m.put("tbConfidence", dto.getTbConfidence()); + m.put("drugResistancePresence", dto.getDrugResistancePresence()); + out.put(order.getId(), m); + } + return out; + } + private DiagnosticOrderResultDto toResultDto(DiagnosticOrder order) { DiagnosticOrderResultDto dto = new DiagnosticOrderResultDto(); dto.setExternalOrderId(order.getExternalOrderId()); diff --git a/src/main/java/com/iemr/flw/service/impl/DynamicFormResponseServiceImpl.java b/src/main/java/com/iemr/flw/service/impl/DynamicFormResponseServiceImpl.java index f74088a6d..b14ab50c1 100644 --- a/src/main/java/com/iemr/flw/service/impl/DynamicFormResponseServiceImpl.java +++ b/src/main/java/com/iemr/flw/service/impl/DynamicFormResponseServiceImpl.java @@ -61,6 +61,8 @@ import java.sql.Timestamp; import java.time.LocalDate; import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.NoSuchElementException; @@ -460,6 +462,24 @@ private QuestionOption resolveOption( // ---- read helpers ---- + @Override + public Map getResponsesByIds(Collection responseIds) { + List responses = formResponseRepo.findAllById(responseIds); + Set versionIds = responses.stream() + .map(FormResponse::getVersionId) + .filter(Objects::nonNull) + .collect(Collectors.toSet()); + Map versionNumberById = formVersionRepo.findAllById(versionIds).stream() + .collect(Collectors.toMap(FormVersion::getVersionId, FormVersion::getVersionNumber)); + Map out = new LinkedHashMap<>(); + for (FormResponse r : responses) { + FormResponseDTO dto = buildFormResponseDTO(r, loadSectionResponseDTOs(r.getResponseId())); + dto.setVersionNumber(versionNumberById.get(r.getVersionId())); + out.put(r.getResponseId(), dto); + } + return out; + } + private List loadSectionResponseDTOs(Long responseId) { List sections = sectionResponseRepo.findByResponseId(responseId); if (sections.isEmpty()) { diff --git a/src/main/java/com/iemr/flw/service/impl/StopTBServiceImpl.java b/src/main/java/com/iemr/flw/service/impl/StopTBServiceImpl.java index c4ceed44b..3c00b1973 100644 --- a/src/main/java/com/iemr/flw/service/impl/StopTBServiceImpl.java +++ b/src/main/java/com/iemr/flw/service/impl/StopTBServiceImpl.java @@ -579,6 +579,61 @@ public Map getAllDiagnostics(Integer providerServiceMapID, Integ return result; } + // ── By id, for the changes feed ─────────────────────────────────────────── + + @Override + public Map> getNurseTBScreeningsByIds(Collection ids) { + Map> out = new LinkedHashMap<>(); + for (TBScreening s : tbScreeningRepo.findAllById(ids)) { + if (!Boolean.TRUE.equals(s.getDeleted())) out.put(s.getId(), screeningToMap(s)); + } + return out; + } + + @Override + public Map> getGeneralExaminationsByIds(Collection ids) { + Map> out = new LinkedHashMap<>(); + for (StopTBGeneralExamination e : generalExaminationRepo.findAllById(ids)) { + if (!Boolean.TRUE.equals(e.getDeleted())) out.put(e.getId(), examToMap(e)); + } + return out; + } + + @Override + public Map> getDiagnosticsByIds(Collection ids) { + Map> out = new LinkedHashMap<>(); + for (StopTBDiagnostics d : diagnosticsRepo.findAllById(ids)) { + if (!Boolean.TRUE.equals(d.getDeleted())) out.put(d.getId(), diagnosticsToMap(d)); + } + return out; + } + + @Override + public Map> getGeneralOpdByVisitIds(Collection benVisitIds) { + Map> out = new LinkedHashMap<>(); + for (Map m : stopTBPrescriptionService.getStandardOpdRecordsByVisitIds(benVisitIds)) { + out.put((Long) m.get("id"), m); + } + return out; + } + + @Override + public Map> getLegacyGeneralOpdByIds(Collection ids) { + List rows = new ArrayList<>(); + Set visitCodes = new HashSet<>(); + for (StopTBGeneralOpd o : generalOpdRepo.findAllById(ids)) { + if (Boolean.TRUE.equals(o.getDeleted())) continue; + rows.add(o); + if (o.getVisitCode() != null) visitCodes.add(o.getVisitCode()); + } + Set standard = stopTBPrescriptionService.visitCodesWithStandardOpd(visitCodes); + Map> out = new LinkedHashMap<>(); + for (StopTBGeneralOpd o : rows) { + if (o.getVisitCode() == null || !standard.contains(o.getVisitCode())) out.put(o.getId(), opdToMap(o)); + } + return out; + } + private Map diagnosticsToMap(StopTBDiagnostics d) { Map m = new LinkedHashMap<>(); m.put("id", d.getId()); diff --git a/src/main/java/com/iemr/flw/service/impl/StopTBSyncServiceImpl.java b/src/main/java/com/iemr/flw/service/impl/StopTBSyncServiceImpl.java new file mode 100644 index 000000000..91b8eda16 --- /dev/null +++ b/src/main/java/com/iemr/flw/service/impl/StopTBSyncServiceImpl.java @@ -0,0 +1,342 @@ +package com.iemr.flw.service.impl; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonParser; +import com.iemr.flw.service.BeneficiaryService; +import com.iemr.flw.service.DiagnosticOrderService; +import com.iemr.flw.service.DynamicFormResponseService; +import com.iemr.flw.service.StopTBService; +import com.iemr.flw.service.StopTBSyncService; +import com.iemr.flw.service.TBConfirmedCaseService; +import com.iemr.flw.service.TBSuspectedService; +import com.iemr.flw.service.sync.StopTBSyncType; +import com.iemr.flw.service.sync.SyncToken; +import jakarta.persistence.EntityManager; +import jakarta.persistence.PersistenceContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.*; + +/** + * STOP-TB changes feed, served only on hubs (camp laptops, and UAT central for testing), where + * AMRIT-DB's hub migrations install t_sync_change_log and its triggers. + * + * The tablet keeps one opaque token. With none it gets a first full download in pages (snapshot); + * after that, each call returns only the records with change-log lines after its token, in their + * current state, plus the next token. Village scope is resolved here through i_ben_flow_outreach, + * the same way today's getAll endpoints do. + * + * @author Piramal Swasthya + */ +@Service +public class StopTBSyncServiceImpl implements StopTBSyncService { + + private final Logger logger = LoggerFactory.getLogger(StopTBSyncServiceImpl.class); + + static final int DEFAULT_LIMIT = 200; + static final int MAX_LIMIT = 500; + /** Lines younger than this may still have earlier seqs uncommitted; they wait for the next call. */ + static final int SAFE_LAG_SECONDS = 5; + /** The hub keeps 30 days of log; a token older than this may point at cleaned-up lines. */ + static final long MAX_TOKEN_AGE_SECONDS = 29L * 24 * 3600; + private static final long SUPPORT_CHECK_TTL_MS = 5 * 60 * 1000L; + private static final String FLOW_TABLE = "db_iemr.i_ben_flow_outreach"; + + // Same date formats the existing getAll endpoints use for these records. + private final Gson stopTbGson = new GsonBuilder().serializeNulls().create(); + + @PersistenceContext(unitName = "db_iemr") + private EntityManager em; + + @Autowired + private BeneficiaryService beneficiaryService; + @Autowired + private StopTBService stopTBService; + @Autowired + private TBSuspectedService tbSuspectedService; + @Autowired + private TBConfirmedCaseService tbConfirmedCaseService; + @Autowired + private DynamicFormResponseService dynamicFormResponseService; + @Autowired + private DiagnosticOrderService diagnosticOrderService; + @Autowired + private ObjectMapper objectMapper; + + private volatile Boolean supported; + private volatile long supportCheckedAt; + + @Override + public Map getChanges(String token, Integer providerServiceMapID, List villageIds, + Integer limit, String authorisation) { + if (!isSupported()) { + Map out = new LinkedHashMap<>(); + out.put("supported", false); + return out; + } + if (providerServiceMapID == null) throw new IllegalArgumentException("providerServiceMapID is required"); + if (villageIds == null || villageIds.isEmpty()) throw new IllegalArgumentException("villageIds is required"); + int pageSize = limit == null || limit <= 0 ? DEFAULT_LIMIT : Math.min(limit, MAX_LIMIT); + + String serverId = serverId(); + SyncToken parsed = SyncToken.parse(token); + long now = System.currentTimeMillis() / 1000; + boolean usable = parsed != null && serverId.equals(parsed.serverId()) + && (parsed.kind() == SyncToken.Kind.SNAPSHOT || now - parsed.issuedAt() <= MAX_TOKEN_AGE_SECONDS); + + Scope scope = new Scope(providerServiceMapID, villageIds); + if (!usable) { + if (parsed != null) logger.info("Sync token {} not usable here, starting a snapshot", parsed); + return snapshotPage(serverId, SyncToken.snapshot(serverId, safePoint(), 0, 0), scope, pageSize, authorisation); + } + return parsed.kind() == SyncToken.Kind.SNAPSHOT + ? snapshotPage(serverId, parsed, scope, pageSize, authorisation) + : deltaPage(serverId, parsed.seq(), scope, pageSize, authorisation); + } + + // ── Snapshot: everything in scope, type by type, keyset-paged ───────────── + + private Map snapshotPage(String serverId, SyncToken at, Scope scope, int pageSize, + String authorisation) { + StopTBSyncType[] types = StopTBSyncType.values(); + int typeIndex = at.typeIndex(); + long after = at.afterKey(); + List> items = new ArrayList<>(); + + while (items.size() < pageSize && typeIndex < types.length) { + StopTBSyncType type = types[typeIndex]; + int want = pageSize - items.size(); + List keys = snapshotKeys(type, scope, after, want); + Map records = build(type, keys, authorisation); + for (Long key : keys) { + String data = records.get(key); + if (data != null) items.add(item(type, "upsert", key, at.seq(), data)); + } + if (keys.size() < want) { + typeIndex++; + after = 0; + } else { + after = keys.get(keys.size() - 1); + break; + } + } + + boolean done = typeIndex >= types.length; + SyncToken next = done + ? SyncToken.delta(serverId, at.seq(), System.currentTimeMillis() / 1000) + : SyncToken.snapshot(serverId, at.seq(), typeIndex, after); + return page(serverId, true, items, next, !done); + } + + @SuppressWarnings("unchecked") + private List snapshotKeys(StopTBSyncType type, Scope scope, long after, int limit) { + List rows = em.createNativeQuery(type.snapshotKeySql()) + .setParameter("psm", scope.psm) + .setParameter("villages", scope.villages) + .setParameter("after", after) + .setParameter("limit", limit) + .getResultList(); + List keys = new ArrayList<>(rows.size()); + for (Object r : rows) keys.add(((Number) r).longValue()); + return keys; + } + + // ── Delta: records with log lines after the token ───────────────────────── + + @SuppressWarnings("unchecked") + private Map deltaPage(String serverId, long from, Scope scope, int pageSize, + String authorisation) { + long safe = Math.max(safePoint(), from); + // One row per record: its latest line in range, and that line's op. Deduplicates a record + // changed many times since the token. + List rows = em.createNativeQuery( + "SELECT l.entity, l.row_pk, l.seq, l.op FROM db_iemr.t_sync_change_log l JOIN (" + + " SELECT c.entity, c.row_pk, MAX(c.seq) AS last_seq FROM db_iemr.t_sync_change_log c " + + " WHERE c.seq > :from AND c.seq <= :safe AND (" + + " (c.ben_reg_id IS NOT NULL AND EXISTS (SELECT 1 FROM db_iemr.i_ben_flow_outreach f " + + " WHERE f.beneficiary_reg_id = c.ben_reg_id AND f.providerServiceMapID = :psm AND f.villageID IN (:villages)))" + + " OR (c.beneficiary_id IS NOT NULL AND EXISTS (SELECT 1 FROM db_iemr.i_ben_flow_outreach f " + + " WHERE f.beneficiary_id = c.beneficiary_id AND f.providerServiceMapID = :psm AND f.villageID IN (:villages))))" + + " GROUP BY c.entity, c.row_pk ORDER BY last_seq LIMIT :limit" + + ") g ON l.seq = g.last_seq ORDER BY l.seq") + .setParameter("from", from) + .setParameter("safe", safe) + .setParameter("psm", scope.psm) + .setParameter("villages", scope.villages) + .setParameter("limit", pageSize) + .getResultList(); + + // Build each type's records in one batch, then emit in log order. + Map> keysByType = new EnumMap<>(StopTBSyncType.class); + for (Object[] r : rows) { + StopTBSyncType type = StopTBSyncType.fromLogEntity((String) r[0]); + if (type != null) keysByType.computeIfAbsent(type, t -> new ArrayList<>()).add(((Number) r[1]).longValue()); + } + Map> built = new EnumMap<>(StopTBSyncType.class); + keysByType.forEach((type, keys) -> built.put(type, build(type, keys, authorisation))); + + List> items = new ArrayList<>(); + long lastSeq = from; + for (Object[] r : rows) { + lastSeq = ((Number) r[2]).longValue(); + StopTBSyncType type = StopTBSyncType.fromLogEntity((String) r[0]); + if (type == null) continue; + long key = ((Number) r[1]).longValue(); + String data = built.get(type).get(key); + String op = r[3] == null ? "U" : r[3].toString(); + if (data != null && !"D".equals(op)) { + items.add(item(type, "upsert", key, lastSeq, data)); + } else if ("D".equals(op) || type != StopTBSyncType.BENEFICIARY) { + // Gone, soft-deleted or no longer a record the app keeps (e.g. an OPD visit with no + // complaint or drug). A beneficiary that just failed to build is never deleted on the + // tablet; only a deleted mapping removes it. + items.add(item(type, "delete", key, lastSeq, null)); + } + } + + boolean hasMore = rows.size() == pageSize; + long nextSeq = hasMore ? lastSeq : safe; + return page(serverId, false, items, SyncToken.delta(serverId, nextSeq, System.currentTimeMillis() / 1000), hasMore); + } + + // ── Records, in the same JSON each type's existing endpoint returns ─────── + + private Map build(StopTBSyncType type, List keys, String authorisation) { + Map out = new HashMap<>(); + if (keys.isEmpty()) return out; + try { + switch (type) { + case BENEFICIARY: + return beneficiaryService.getBeneficiaryRecordsByRegIds(keys, authorisation); + case TB_SCREENING: + stopTBService.getNurseTBScreeningsByIds(keys).forEach((k, m) -> out.put(k, stopTbGson.toJson(m))); + return out; + case GENERAL_EXAMINATION: + stopTBService.getGeneralExaminationsByIds(keys).forEach((k, m) -> out.put(k, stopTbGson.toJson(m))); + return out; + case GENERAL_OPD: + stopTBService.getGeneralOpdByVisitIds(keys).forEach((k, m) -> out.put(k, stopTbGson.toJson(m))); + return out; + case GENERAL_OPD_LEGACY: + stopTBService.getLegacyGeneralOpdByIds(keys).forEach((k, m) -> out.put(k, stopTbGson.toJson(m))); + return out; + case DIAGNOSTICS: + stopTBService.getDiagnosticsByIds(keys).forEach((k, m) -> out.put(k, stopTbGson.toJson(m))); + return out; + case TB_SUSPECTED: + return tbSuspectedService.getRecordsByIds(keys); + case TB_CONFIRMED: + return tbConfirmedCaseService.getRecordsByIds(keys); + case FORM_RESPONSE: + for (Map.Entry e : dynamicFormResponseService.getResponsesByIds(keys).entrySet()) { + out.put(e.getKey(), objectMapper.writeValueAsString(e.getValue())); + } + return out; + case DIAGNOSTIC_ORDER: + for (Map.Entry> e : diagnosticOrderService.getOrderRecordsByIds(keys).entrySet()) { + out.put(e.getKey(), objectMapper.writeValueAsString(e.getValue())); + } + return out; + default: + return out; + } + } catch (Exception e) { + // A failed batch must not turn into deletes on the tablet: fail the whole call, so the + // token doesn't move and the tablet retries. + throw new IllegalStateException("Could not build " + type.apiName() + " records: " + e.getMessage(), e); + } + } + + private Map item(StopTBSyncType type, String op, long key, long version, String data) { + Map m = new LinkedHashMap<>(); + m.put("type", type.apiName()); + m.put("op", op); + m.put("key", key); + m.put("version", version); + // The record exactly as the type's existing endpoint returns it, so the app reuses its parser. + m.put("data", data == null ? null : JsonParser.parseString(data)); + return m; + } + + private Map page(String serverId, boolean snapshot, List> items, + SyncToken next, boolean hasMore) { + Map out = new LinkedHashMap<>(); + out.put("supported", true); + out.put("serverId", serverId); + out.put("snapshot", snapshot); + out.put("items", items); + out.put("nextToken", next.encode()); + out.put("hasMore", hasMore); + return out; + } + + // ── Hub state ───────────────────────────────────────────────────────────── + + private String serverId() { + return String.valueOf(em.createNativeQuery("SELECT @@server_uuid").getSingleResult()); + } + + private long safePoint() { + Object r = em.createNativeQuery("SELECT COALESCE(MAX(seq), 0) FROM db_iemr.t_sync_change_log " + + "WHERE changed_at <= NOW(6) - INTERVAL " + SAFE_LAG_SECONDS + " SECOND") + .getSingleResult(); + return ((Number) r).longValue(); + } + + /** + * True only where the change log exists AND every source table has its insert, update and delete + * triggers. Otherwise a change could be missed silently, so the feed reports itself unsupported and + * the app keeps using the full pull. + */ + @SuppressWarnings("unchecked") + boolean isSupported() { + long now = System.currentTimeMillis(); + if (supported != null && now - supportCheckedAt < SUPPORT_CHECK_TTL_MS) return supported; + boolean ok; + try { + Number logTables = (Number) em.createNativeQuery("SELECT COUNT(*) FROM information_schema.tables " + + "WHERE table_schema = 'db_iemr' AND table_name = 't_sync_change_log'").getSingleResult(); + Set withAllTriggers = new HashSet<>(); + List rows = em.createNativeQuery("SELECT LOWER(CONCAT(event_object_schema, '.', event_object_table)), " + + "COUNT(DISTINCT event_manipulation) FROM information_schema.triggers " + + "WHERE trigger_name LIKE 'trg\\_scl\\_%' GROUP BY event_object_schema, event_object_table").getResultList(); + for (Object[] r : rows) { + String table = (String) r[0]; + // The flow table only decides scope: insert and update are enough (rows are never deleted). + int needed = FLOW_TABLE.equals(table) ? 2 : 3; + if (((Number) r[1]).intValue() >= needed) withAllTriggers.add(table); + } + List missing = new ArrayList<>(); + for (StopTBSyncType type : StopTBSyncType.values()) { + for (String table : type.sourceTables()) { + if (!withAllTriggers.contains(table)) missing.add(table); + } + } + ok = logTables.intValue() > 0 && missing.isEmpty(); + if (logTables.intValue() > 0 && !missing.isEmpty()) { + logger.error("STOP-TB changes feed disabled: change-log triggers missing on {}", missing); + } + } catch (Exception e) { + logger.warn("STOP-TB changes feed: could not check the change log: {}", e.getMessage()); + ok = false; + } + supported = ok; + supportCheckedAt = now; + return ok; + } + + private static final class Scope { + final Integer psm; + final List villages; + + Scope(Integer psm, List villages) { + this.psm = psm; + this.villages = villages; + } + } +} diff --git a/src/main/java/com/iemr/flw/service/impl/TBConfirmedCaseServiceImpl.java b/src/main/java/com/iemr/flw/service/impl/TBConfirmedCaseServiceImpl.java index 836d853ac..2db58ede3 100644 --- a/src/main/java/com/iemr/flw/service/impl/TBConfirmedCaseServiceImpl.java +++ b/src/main/java/com/iemr/flw/service/impl/TBConfirmedCaseServiceImpl.java @@ -204,9 +204,25 @@ public String getByProviderServiceMapId(Integer providerServiceMapID, Integer vi return buildTbConfirmedCasesResponse(null, list); } - private String buildTbConfirmedCasesResponse(Integer userId, List list) { - List dtoList = list.stream().map(this::toDTO).collect(Collectors.toList()); + @Override + public Map getRecordsByIds(java.util.Collection ids) { + List intIds = ids.stream().map(Long::intValue).collect(Collectors.toList()); + List dtoList = withCounselled( + repository.findAllById(intIds).stream().map(this::toDTO).collect(Collectors.toList())); + Gson gson = confirmedGson(); + Map out = new java.util.LinkedHashMap<>(); + for (TBConfirmedCaseDTO dto : dtoList) out.put(dto.getId().longValue(), gson.toJson(dto)); + return out; + } + + private Gson confirmedGson() { + return new GsonBuilder() + .registerTypeAdapter(LocalDate.class, new LocalDateAdapter()) + .setDateFormat("MMM dd, yyyy h:mm:ss a") + .create(); + } + private List withCounselled(List dtoList) { List benIds = dtoList.stream().map(TBConfirmedCaseDTO::getBenId).collect(Collectors.toList()); Set counselledBenIds = Collections.emptySet(); if (!benIds.isEmpty()) { @@ -217,15 +233,16 @@ private String buildTbConfirmedCasesResponse(Integer userId, List counselled = counselledBenIds; dtoList.forEach(dto -> dto.setCounselled(counselled.contains(dto.getBenId()))); + return dtoList; + } + + private String buildTbConfirmedCasesResponse(Integer userId, List list) { + List dtoList = withCounselled(list.stream().map(this::toDTO).collect(Collectors.toList())); Map response = new java.util.HashMap<>(); response.put("userId", userId); response.put("tbConfirmedCases", dtoList); - Gson gson = new GsonBuilder() - .registerTypeAdapter(LocalDate.class, new LocalDateAdapter()) - .setDateFormat("MMM dd, yyyy h:mm:ss a") - .create(); - return gson.toJson(response); + return confirmedGson().toJson(response); } // Utility: convert entity -> DTO diff --git a/src/main/java/com/iemr/flw/service/impl/TBSuspectedServiceImpl.java b/src/main/java/com/iemr/flw/service/impl/TBSuspectedServiceImpl.java index 99db4fcbe..1f4eca766 100644 --- a/src/main/java/com/iemr/flw/service/impl/TBSuspectedServiceImpl.java +++ b/src/main/java/com/iemr/flw/service/impl/TBSuspectedServiceImpl.java @@ -165,4 +165,26 @@ public String getByUserId(GetBenRequestHandler request) { return gson.toJson(tbSuspectedRequestDTO); } + @Override + public java.util.Map getRecordsByIds(java.util.Collection ids) { + Gson gson = new GsonBuilder().setDateFormat("MMM dd, yyyy h:mm:ss a").create(); + java.util.Map out = new java.util.LinkedHashMap<>(); + for (TBSuspected tbSuspected : tbSuspectedRepo.findAllById(ids)) { + TBSuspectedDTO dto = modelMapper.map(tbSuspected, TBSuspectedDTO.class); + dto.setUpdateDate(tbSuspected.getLastModDate()); + dto.setUpdatedBy(tbSuspected.getModifiedBy()); + out.put(tbSuspected.getId(), gson.toJson(dto)); + // Same side effect as getByUserId, so incentives don't depend on which pull the app uses. + if (Boolean.TRUE.equals(tbSuspected.getIsConfirmed())) { + incentiveLogicService.incentiveForTbSuspected( + tbSuspected.getBenId(), + tbSuspected.getVisitDate(), + tbSuspected.getVisitDate(), + tbSuspected.getUserId() + ); + } + } + return out; + } + } diff --git a/src/main/java/com/iemr/flw/service/sync/StopTBSyncType.java b/src/main/java/com/iemr/flw/service/sync/StopTBSyncType.java new file mode 100644 index 000000000..d4c8308f8 --- /dev/null +++ b/src/main/java/com/iemr/flw/service/sync/StopTBSyncType.java @@ -0,0 +1,128 @@ +package com.iemr.flw.service.sync; + +import java.util.List; + +/** + * The record types the STOP-TB changes feed (/stoptb/sync/changes) serves, in snapshot order + * (beneficiaries first, so a tablet has the person before their TB records). + * + * For each type: + * - {@code logEntity}: the entity name the hub's change-log triggers write (AMRIT-DB + * db/migration/dbiemr_hub and dbidentity_hub); + * - {@code snapshotKeySql}: the keys in scope, for the first full download, with the same filters + * the type's getAll endpoint applies. Parameters: :psm, :villages, :after, :limit; + * - {@code sourceTables}: every table the type's record is built from. Each must have the change-log + * triggers, or edits to it would never reach tablets. The feed checks this on the hub + * ({@code StopTBSyncServiceImpl#isSupported}) and a unit test checks every mapped table is either + * here or deliberately not synced. + * + * @author Piramal Swasthya + */ +public enum StopTBSyncType { + + BENEFICIARY("beneficiary", "beneficiary", + "SELECT DISTINCT f.beneficiary_reg_id FROM db_iemr.i_ben_flow_outreach f " + + "WHERE f.providerServiceMapID = :psm AND f.villageID IN (:villages) AND f.deleted = 0 " + + "AND f.beneficiary_reg_id > :after ORDER BY f.beneficiary_reg_id LIMIT :limit", + List.of("db_identity.i_beneficiarymapping", "db_identity.i_beneficiarydetails", + "db_identity.i_beneficiarydetails_rmnch", "db_identity.i_bornbirthdeatils", + "db_identity.i_beneficiaryaddress", "db_identity.i_beneficiarycontacts", + "db_identity.i_beneficiaryimage", "db_identity.i_beneficiaryaccount", + "db_identity.i_householddetails", "db_iemr.t_phy_anthropometry", "db_iemr.t_phy_vitals", + "db_iemr.i_ben_flow_outreach")), + + TB_SCREENING("tbScreening", "tb_screening", + "SELECT s.id FROM db_iemr.tb_screening s WHERE (s.deleted = 0 OR s.deleted IS NULL) " + + "AND s.ben_reg_id IN (" + Scope.REG + ") AND s.id > :after ORDER BY s.id LIMIT :limit", + List.of("db_iemr.tb_screening")), + + GENERAL_EXAMINATION("generalExamination", "general_examination", + "SELECT e.id FROM db_iemr.tb_stoptb_general_examination e WHERE e.deleted = 0 " + + "AND e.beneficiary_reg_id IN (" + Scope.REG + ") AND e.id > :after ORDER BY e.id LIMIT :limit", + List.of("db_iemr.tb_stoptb_general_examination")), + + /** OPD visits saved to the standard tables, keyed by BenVisitID. */ + GENERAL_OPD("generalOpd", "opd_visit", + "SELECT v.BenVisitID FROM db_iemr.t_benvisitdetail v WHERE v.Deleted = 0 AND v.ProviderServiceMapID = :psm " + + "AND v.BeneficiaryRegID IN (" + Scope.REG + ") AND v.BenVisitID > :after ORDER BY v.BenVisitID LIMIT :limit", + List.of("db_iemr.t_benvisitdetail", "db_iemr.t_benchiefcomplaint", "db_iemr.t_prescribeddrug")), + + /** Older free-text OPD rows; same record shape as GENERAL_OPD. */ + GENERAL_OPD_LEGACY("generalOpdLegacy", "general_opd", + "SELECT o.id FROM db_iemr.tb_stoptb_general_opd o WHERE o.deleted = 0 " + + "AND o.ben_reg_id IN (" + Scope.REG + ") AND o.id > :after ORDER BY o.id LIMIT :limit", + List.of("db_iemr.tb_stoptb_general_opd")), + + DIAGNOSTICS("diagnostics", "diagnostics", + "SELECT d.id FROM db_iemr.tb_stoptb_diagnostics d WHERE d.deleted = 0 " + + "AND d.ben_reg_id IN (" + Scope.REG + ") AND d.id > :after ORDER BY d.id LIMIT :limit", + List.of("db_iemr.tb_stoptb_diagnostics")), + + TB_SUSPECTED("tbSuspected", "tb_suspected", + "SELECT t.id FROM db_iemr.tb_suspected t WHERE t.ben_id IN (" + Scope.BEN_ID + ") " + + "AND t.id > :after ORDER BY t.id LIMIT :limit", + List.of("db_iemr.tb_suspected")), + + TB_CONFIRMED("tbConfirmed", "tb_confirmed", + "SELECT c.id FROM db_iemr.tb_confirmed_cases c WHERE c.ben_id IN (" + Scope.BEN_ID + ") " + + "AND c.id > :after ORDER BY c.id LIMIT :limit", + List.of("db_iemr.tb_confirmed_cases")), + + FORM_RESPONSE("formResponse", "form_response", + "SELECT r.responseId FROM db_iemr.t_form_response r WHERE r.beneficiaryId IN (" + Scope.BEN_ID_NOT_DELETED + ") " + + "AND r.responseId > :after ORDER BY r.responseId LIMIT :limit", + List.of("db_iemr.t_form_response", "db_iemr.t_section_response", "db_iemr.t_question_response")), + + DIAGNOSTIC_ORDER("diagnosticOrder", "diagnostic_order", + "SELECT o.id FROM db_iemr.tb_diagnostic_order o WHERE o.deleted = 0 " + + "AND o.beneficiary_id IN (" + Scope.BEN_ID_NOT_DELETED + ") AND o.id > :after ORDER BY o.id LIMIT :limit", + List.of("db_iemr.tb_diagnostic_order", "db_iemr.tb_diagnostic_result")); + + /** Beneficiaries in the requested villages, the same way today's getAll endpoints scope. */ + static final class Scope { + static final String REG = "SELECT f.beneficiary_reg_id FROM db_iemr.i_ben_flow_outreach f " + + "WHERE f.providerServiceMapID = :psm AND f.villageID IN (:villages)"; + static final String BEN_ID = "SELECT f.beneficiary_id FROM db_iemr.i_ben_flow_outreach f " + + "WHERE f.providerServiceMapID = :psm AND f.villageID IN (:villages)"; + static final String BEN_ID_NOT_DELETED = BEN_ID + " AND f.deleted = 0"; + + private Scope() { + } + } + + private final String apiName; + private final String logEntity; + private final String snapshotKeySql; + private final List sourceTables; + + StopTBSyncType(String apiName, String logEntity, String snapshotKeySql, List sourceTables) { + this.apiName = apiName; + this.logEntity = logEntity; + this.snapshotKeySql = snapshotKeySql; + this.sourceTables = sourceTables; + } + + /** The "type" field of an item in the feed. */ + public String apiName() { + return apiName; + } + + public String logEntity() { + return logEntity; + } + + public String snapshotKeySql() { + return snapshotKeySql; + } + + public List sourceTables() { + return sourceTables; + } + + public static StopTBSyncType fromLogEntity(String entity) { + for (StopTBSyncType t : values()) { + if (t.logEntity.equals(entity)) return t; + } + return null; + } +} diff --git a/src/main/java/com/iemr/flw/service/sync/SyncToken.java b/src/main/java/com/iemr/flw/service/sync/SyncToken.java new file mode 100644 index 000000000..d0f133b02 --- /dev/null +++ b/src/main/java/com/iemr/flw/service/sync/SyncToken.java @@ -0,0 +1,97 @@ +package com.iemr.flw.service.sync; + +/** + * The tablet's opaque sync position. Two kinds: + * + * d.<serverId>.<seq>.<issuedEpochSeconds> + * changes after change-log line seq on this hub; + * s.<serverId>.<snapshotSeq>.<typeIndex>.<afterKey> + * part-way through a first download: the next page starts at type typeIndex, after afterKey, + * and when it finishes the tablet continues with changes after snapshotSeq. + * + * serverId is the hub's MySQL server_uuid, so a token from another hub is never trusted. The issue + * time lets the hub tell when the log lines a token points after may have been cleaned up. + * + * @author Piramal Swasthya + */ +public final class SyncToken { + + public enum Kind { DELTA, SNAPSHOT } + + private final Kind kind; + private final String serverId; + private final long seq; + private final long issuedAt; + private final int typeIndex; + private final long afterKey; + + private SyncToken(Kind kind, String serverId, long seq, long issuedAt, int typeIndex, long afterKey) { + this.kind = kind; + this.serverId = serverId; + this.seq = seq; + this.issuedAt = issuedAt; + this.typeIndex = typeIndex; + this.afterKey = afterKey; + } + + public static SyncToken delta(String serverId, long seq, long issuedAt) { + return new SyncToken(Kind.DELTA, serverId, seq, issuedAt, 0, 0); + } + + public static SyncToken snapshot(String serverId, long snapshotSeq, int typeIndex, long afterKey) { + return new SyncToken(Kind.SNAPSHOT, serverId, snapshotSeq, 0, typeIndex, afterKey); + } + + /** null for a missing or unreadable token; the caller then starts a snapshot. */ + public static SyncToken parse(String token) { + if (token == null || token.isBlank()) return null; + String[] p = token.trim().split("\\."); + try { + if (p.length == 4 && "d".equals(p[0]) && !p[1].isEmpty()) { + return delta(p[1], Long.parseLong(p[2]), Long.parseLong(p[3])); + } + if (p.length == 5 && "s".equals(p[0]) && !p[1].isEmpty()) { + return snapshot(p[1], Long.parseLong(p[2]), Integer.parseInt(p[3]), Long.parseLong(p[4])); + } + } catch (NumberFormatException e) { + return null; + } + return null; + } + + public String encode() { + return kind == Kind.DELTA + ? "d." + serverId + "." + seq + "." + issuedAt + : "s." + serverId + "." + seq + "." + typeIndex + "." + afterKey; + } + + public Kind kind() { + return kind; + } + + public String serverId() { + return serverId; + } + + /** DELTA: last change-log line applied. SNAPSHOT: the line changes continue from afterwards. */ + public long seq() { + return seq; + } + + public long issuedAt() { + return issuedAt; + } + + public int typeIndex() { + return typeIndex; + } + + public long afterKey() { + return afterKey; + } + + @Override + public String toString() { + return encode(); + } +} diff --git a/src/test/java/com/iemr/flw/service/sync/StopTBSyncCoverageTest.java b/src/test/java/com/iemr/flw/service/sync/StopTBSyncCoverageTest.java new file mode 100644 index 000000000..161719b63 --- /dev/null +++ b/src/test/java/com/iemr/flw/service/sync/StopTBSyncCoverageTest.java @@ -0,0 +1,107 @@ +package com.iemr.flw.service.sync; + +import jakarta.persistence.Entity; +import jakarta.persistence.Table; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.config.BeanDefinition; +import org.springframework.context.annotation.ClassPathScanningCandidateComponentProvider; +import org.springframework.core.type.filter.AnnotationTypeFilter; + +import java.util.*; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Every table FLW-API maps must be either synced to STOP-TB tablets through the hub change log + * (listed in a {@link StopTBSyncType}'s sourceTables, which the hub checks for triggers) or listed + * below as deliberately not synced. A new entity fails this test until someone decides which, so a + * STOP-TB table can't be added without its change-log triggers. + */ +class StopTBSyncCoverageTest { + + /** STOP-TB-adjacent tables that the app does not pull, with the reason. */ + static final Map NOT_SYNCED_STOPTB = Map.ofEntries( + Map.entry("db_iemr.t_phy_generalexam", "copy of the general examination for the standard tables; the app pulls tb_stoptb_general_examination"), + Map.entry("db_iemr.t_benreferdetails", "copy written on save for the standard tables; not read by any STOP-TB pull"), + Map.entry("db_iemr.t_prescription", "OPD drugs are read from t_prescribeddrug, which is synced"), + Map.entry("db_iemr.tb_stoptb_registration", "not read by any service"), + Map.entry("db_iemr.tb_diagnostic_document", "the app does not pull documents"), + Map.entry("db_iemr.tb_diagnostic_provider_token", "vendor credentials, server only"), + Map.entry("db_identity.m_beneficiaryregidmapping", "BenRegId/BeneficiaryID pool, fixed once assigned"), + Map.entry("db_iemr.t_dynamic_form", "form definitions: master data, pulled separately"), + Map.entry("db_iemr.t_form_version", "form definitions: master data, pulled separately"), + Map.entry("db_iemr.t_form_section", "form definitions: master data, pulled separately"), + Map.entry("db_iemr.t_section_question", "form definitions: master data, pulled separately"), + Map.entry("db_iemr.t_question_option", "form definitions: master data, pulled separately"), + Map.entry("db_iemr.t_question_validation", "form definitions: master data, pulled separately"), + Map.entry("db_iemr.t_option_condition", "form definitions: master data, pulled separately"), + Map.entry("db_iemr.t_dynamic_form_migration_log", "server-side bookkeeping"), + Map.entry("db_iemr.m_user", "user master, read at login"), + Map.entry("db_iemr.v_userservicerolemapping", "view of user roles, read at login"), + Map.entry("db_iemr.general_opd_entry", "not part of the STOP-TB pull")); + + /** Tables of other FLW programmes (ASHA app), not used by STOP-TB. */ + static final Set OTHER_PROGRAMMES = Set.of( + "db_iemr.adolescent_health", "db_iemr.ahd_form", "db_iemr.anc_counselling_care", "db_iemr.asha_profile", + "db_iemr.asha_supervisor_mapping", "db_iemr.campaign_filariasis_mda", "db_iemr.campaign_ors", + "db_iemr.campaign_pulse_polio", "db_iemr.cdtf_visit_details", "db_iemr.deworming_form", + "db_iemr.eligible_couple_tracking", "db_iemr.high_risk_assess", "db_iemr.i_mobilization_mosquito_net", + "db_iemr.incentive_activity_record", "db_iemr.incentive_pending_activity", "db_iemr.irs_round", + "db_iemr.leprosy_follow_up", "db_iemr.m_immunizationservicevaccination", "db_iemr.m_incentive_activity", + "db_iemr.m_incentive_activity_lang_mapping", "db_iemr.m_otp_beneficiary", "db_iemr.m_smstemplate", + "db_iemr.maa_meeting", "db_iemr.malaria_follow_up", "db_iemr.non_pregnant_high_risk_assess", + "db_iemr.non_pregnant_high_risk_track", "db_iemr.phc_review_meeting", "db_iemr.pregnant_high_risk_assess", + "db_iemr.pregnant_high_risk_track", "db_iemr.sammelan_attachment", "db_iemr.sammelan_record", "db_iemr.screening_aesje", + "db_iemr.screening_filaria", "db_iemr.screening_kala_azar", "db_iemr.screening_leprosy", + "db_iemr.screening_malaria", "db_iemr.t_anc_visit", "db_iemr.t_anccare", "db_iemr.t_cbacdetails", + "db_iemr.t_cdr", "db_iemr.t_child_register", "db_iemr.t_childvaccinedetail1", "db_iemr.t_delivery_outcome", + "db_iemr.t_eligible_couple_register", "db_iemr.t_eye_checkup", "db_iemr.t_hbnc_part1", "db_iemr.t_hbnc_part2", + "db_iemr.t_hbnc_visit", "db_iemr.t_hbnc_visit_card", "db_iemr.t_hbyc", "db_iemr.t_hbyc_child_visits", + "db_iemr.t_ifa_distribution", "db_iemr.t_ifa_distribution_data", "db_iemr.t_infant_register", + "db_iemr.t_mda_distribution_data", "db_iemr.t_mdsr", "db_iemr.t_micro_birth_plan", "db_iemr.t_ors_distribution", + "db_iemr.t_pmsma", "db_iemr.t_pnc_visit", "db_iemr.t_pnccare", "db_iemr.t_pregnant_woman_register", + "db_iemr.t_sam_visit", "db_iemr.uwin_session_record", "db_iemr.vhnc_form", "db_iemr.vhnd_form", + "db_identity.i_cbac_additional_details", "db_identity.i_cbacdetails"); + + @Test + void everyMappedTableIsSyncedOrDeliberatelyNot() throws Exception { + Set synced = new HashSet<>(); + for (StopTBSyncType t : StopTBSyncType.values()) synced.addAll(t.sourceTables()); + + List unclassified = new ArrayList<>(); + for (String table : mappedTables()) { + if (!synced.contains(table) && !NOT_SYNCED_STOPTB.containsKey(table) && !OTHER_PROGRAMMES.contains(table)) { + unclassified.add(table); + } + } + Collections.sort(unclassified); + assertTrue(unclassified.isEmpty(), "Tables neither synced to STOP-TB tablets nor listed as not synced: " + + unclassified + ". If the STOP-TB app reads it, add it to a StopTBSyncType's sourceTables and add " + + "change-log triggers in AMRIT-DB db/migration/*_hub; otherwise list it here with a reason."); + } + + @Test + void syncedTablesAreNotAlsoListedAsNotSynced() { + for (StopTBSyncType t : StopTBSyncType.values()) { + for (String table : t.sourceTables()) { + assertTrue(!NOT_SYNCED_STOPTB.containsKey(table) && !OTHER_PROGRAMMES.contains(table), table); + } + } + } + + /** schema.table for every @Entity under com.iemr.flw.domain, lower-case. */ + static Set mappedTables() throws ClassNotFoundException { + ClassPathScanningCandidateComponentProvider scanner = new ClassPathScanningCandidateComponentProvider(false); + scanner.addIncludeFilter(new AnnotationTypeFilter(Entity.class)); + Set tables = new TreeSet<>(); + for (BeanDefinition bd : scanner.findCandidateComponents("com.iemr.flw.domain")) { + Class c = Class.forName(bd.getBeanClassName()); + Table t = c.getAnnotation(Table.class); + String name = t != null && !t.name().isEmpty() ? t.name() : c.getSimpleName(); + String schema = t != null && !t.schema().isEmpty() ? t.schema() + : c.getPackageName().endsWith(".identity") ? "db_identity" : "db_iemr"; + tables.add((schema + "." + name).toLowerCase(Locale.ROOT)); + } + return tables; + } +} diff --git a/src/test/java/com/iemr/flw/service/sync/SyncTokenTest.java b/src/test/java/com/iemr/flw/service/sync/SyncTokenTest.java new file mode 100644 index 000000000..968b21f03 --- /dev/null +++ b/src/test/java/com/iemr/flw/service/sync/SyncTokenTest.java @@ -0,0 +1,46 @@ +package com.iemr.flw.service.sync; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +class SyncTokenTest { + + private static final String UUID = "3f2a6c1e-1b2c-11ef-9f1a-0242ac120002"; + + @Test + void deltaRoundTrips() { + SyncToken t = SyncToken.parse(SyncToken.delta(UUID, 1000, 1761700000L).encode()); + assertEquals(SyncToken.Kind.DELTA, t.kind()); + assertEquals(UUID, t.serverId()); + assertEquals(1000, t.seq()); + assertEquals(1761700000L, t.issuedAt()); + } + + @Test + void snapshotRoundTrips() { + SyncToken t = SyncToken.parse(SyncToken.snapshot(UUID, 5230, 3, 900).encode()); + assertEquals(SyncToken.Kind.SNAPSHOT, t.kind()); + assertEquals(5230, t.seq()); + assertEquals(3, t.typeIndex()); + assertEquals(900, t.afterKey()); + } + + @Test + void unreadableTokensMeanStartOver() { + assertNull(SyncToken.parse(null)); + assertNull(SyncToken.parse("")); + assertNull(SyncToken.parse("garbage")); + assertNull(SyncToken.parse("d." + UUID + ".x.1")); + assertNull(SyncToken.parse("d..1.1")); + assertNull(SyncToken.parse("q." + UUID + ".1.1")); + } + + @Test + void everyLogEntityMapsToOneType() { + for (StopTBSyncType t : StopTBSyncType.values()) { + assertSame(t, StopTBSyncType.fromLogEntity(t.logEntity())); + } + assertNull(StopTBSyncType.fromLogEntity("unknown")); + } +} From beb4d394d3b014dcc3e394f2187355d4a4c15fb6 Mon Sep 17 00:00:00 2001 From: Mithun James <1007084+drtechie@users.noreply.github.com> Date: Thu, 29 Oct 2026 11:24:26 +0530 Subject: [PATCH 02/12] feat(stoptb): let the app choose which record types the changes feed returns Co-Authored-By: Claude Opus 5.5 --- .../flw/controller/StopTBSyncController.java | 11 ++++-- .../iemr/flw/service/StopTBSyncService.java | 4 ++- .../service/impl/StopTBSyncServiceImpl.java | 35 ++++++++++++++++--- 3 files changed, 43 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/iemr/flw/controller/StopTBSyncController.java b/src/main/java/com/iemr/flw/controller/StopTBSyncController.java index 9c1429769..9c1a8764f 100644 --- a/src/main/java/com/iemr/flw/controller/StopTBSyncController.java +++ b/src/main/java/com/iemr/flw/controller/StopTBSyncController.java @@ -26,7 +26,8 @@ public class StopTBSyncController { private StopTBSyncService stopTBSyncService; /** - * Body: { token: string|null, providerServiceMapID: int, villageIds: [int], limit: int }. + * Body: { token: string|null, providerServiceMapID: int, villageIds: [int], types: [string], limit: int }. + * types: the record types the app applies (all when omitted); changing it requires a new snapshot. * Response: { supported, serverId, snapshot, items: [{type, op, key, version, data}], nextToken, hasMore }. * Call again with nextToken while hasMore is true. On a server without the hub change log the * response is { supported: false } and the app keeps using the getAll endpoints. @@ -46,9 +47,15 @@ public String getChanges(@RequestBody Map body, if (v != null) villageIds.add((int) Double.parseDouble(v.toString())); } } + List types = new ArrayList<>(); + if (body.get("types") instanceof List) { + for (Object t : (List) body.get("types")) { + if (t != null) types.add(t.toString()); + } + } Integer limit = body.get("limit") != null ? (int) Double.parseDouble(body.get("limit").toString()) : null; Map data = stopTBSyncService.getChanges( - token != null ? token.toString() : null, psm, villageIds, limit, authorization); + token != null ? token.toString() : null, psm, villageIds, types, limit, authorization); response.setResponse(gson.toJson(data)); } catch (Exception e) { logger.error("Error in sync/changes: " + e); diff --git a/src/main/java/com/iemr/flw/service/StopTBSyncService.java b/src/main/java/com/iemr/flw/service/StopTBSyncService.java index 81fa47c17..580542fe6 100644 --- a/src/main/java/com/iemr/flw/service/StopTBSyncService.java +++ b/src/main/java/com/iemr/flw/service/StopTBSyncService.java @@ -9,7 +9,9 @@ public interface StopTBSyncService { * One page of the STOP-TB changes feed for the given villages: a first full download when the * token is missing, from another hub or too old; otherwise only what changed after it. * Returns {supported:false} where the hub change log isn't installed, so the app keeps its full pull. + * types limits the feed to the record types the app applies (StopTBSyncType api names); null or + * empty means all. The app must start a new snapshot (no token) whenever it changes this list. */ Map getChanges(String token, Integer providerServiceMapID, List villageIds, - Integer limit, String authorisation); + List types, Integer limit, String authorisation); } diff --git a/src/main/java/com/iemr/flw/service/impl/StopTBSyncServiceImpl.java b/src/main/java/com/iemr/flw/service/impl/StopTBSyncServiceImpl.java index 91b8eda16..2d8bb51d5 100644 --- a/src/main/java/com/iemr/flw/service/impl/StopTBSyncServiceImpl.java +++ b/src/main/java/com/iemr/flw/service/impl/StopTBSyncServiceImpl.java @@ -73,7 +73,7 @@ public class StopTBSyncServiceImpl implements StopTBSyncService { @Override public Map getChanges(String token, Integer providerServiceMapID, List villageIds, - Integer limit, String authorisation) { + List types, Integer limit, String authorisation) { if (!isSupported()) { Map out = new LinkedHashMap<>(); out.put("supported", false); @@ -89,7 +89,7 @@ public Map getChanges(String token, Integer providerServiceMapID boolean usable = parsed != null && serverId.equals(parsed.serverId()) && (parsed.kind() == SyncToken.Kind.SNAPSHOT || now - parsed.issuedAt() <= MAX_TOKEN_AGE_SECONDS); - Scope scope = new Scope(providerServiceMapID, villageIds); + Scope scope = new Scope(providerServiceMapID, villageIds, requestedTypes(types)); if (!usable) { if (parsed != null) logger.info("Sync token {} not usable here, starting a snapshot", parsed); return snapshotPage(serverId, SyncToken.snapshot(serverId, safePoint(), 0, 0), scope, pageSize, authorisation); @@ -110,6 +110,11 @@ private Map snapshotPage(String serverId, SyncToken at, Scope sc while (items.size() < pageSize && typeIndex < types.length) { StopTBSyncType type = types[typeIndex]; + if (!scope.types.contains(type)) { + typeIndex++; + after = 0; + continue; + } int want = pageSize - items.size(); List keys = snapshotKeys(type, scope, after, want); Map records = build(type, keys, authorisation); @@ -157,7 +162,7 @@ private Map deltaPage(String serverId, long from, Scope scope, i List rows = em.createNativeQuery( "SELECT l.entity, l.row_pk, l.seq, l.op FROM db_iemr.t_sync_change_log l JOIN (" + " SELECT c.entity, c.row_pk, MAX(c.seq) AS last_seq FROM db_iemr.t_sync_change_log c " - + " WHERE c.seq > :from AND c.seq <= :safe AND (" + + " WHERE c.seq > :from AND c.seq <= :safe AND c.entity IN (:entities) AND (" + " (c.ben_reg_id IS NOT NULL AND EXISTS (SELECT 1 FROM db_iemr.i_ben_flow_outreach f " + " WHERE f.beneficiary_reg_id = c.ben_reg_id AND f.providerServiceMapID = :psm AND f.villageID IN (:villages)))" + " OR (c.beneficiary_id IS NOT NULL AND EXISTS (SELECT 1 FROM db_iemr.i_ben_flow_outreach f " @@ -166,6 +171,7 @@ private Map deltaPage(String serverId, long from, Scope scope, i + ") g ON l.seq = g.last_seq ORDER BY l.seq") .setParameter("from", from) .setParameter("safe", safe) + .setParameter("entities", scope.logEntities()) .setParameter("psm", scope.psm) .setParameter("villages", scope.villages) .setParameter("limit", pageSize) @@ -330,13 +336,34 @@ boolean isSupported() { return ok; } + private static Set requestedTypes(List names) { + Set out = EnumSet.noneOf(StopTBSyncType.class); + if (names == null || names.isEmpty()) { + out.addAll(EnumSet.allOf(StopTBSyncType.class)); + return out; + } + for (StopTBSyncType t : StopTBSyncType.values()) { + if (names.contains(t.apiName())) out.add(t); + } + if (out.isEmpty()) throw new IllegalArgumentException("No known record types in " + names); + return out; + } + private static final class Scope { final Integer psm; final List villages; + final Set types; - Scope(Integer psm, List villages) { + Scope(Integer psm, List villages, Set types) { this.psm = psm; this.villages = villages; + this.types = types; + } + + List logEntities() { + List out = new ArrayList<>(); + for (StopTBSyncType t : types) out.add(t.logEntity()); + return out; } } } From 8143a751611cb57e8706221ddeb0da18aa76fdc1 Mon Sep 17 00:00:00 2001 From: Mithun James <1007084+drtechie@users.noreply.github.com> Date: Thu, 29 Oct 2026 11:56:42 +0530 Subject: [PATCH 03/12] fix(stoptb): treat t_prescription as a source of the OPD record The OPD record's notes come from the visit's latest t_prescription instruction. A remarks-only save writes only that table, so without it in the list the change log missed the edit and tablets kept the old note. Paired with the t_prescription trigger in the AMRIT-DB hub migration. Co-Authored-By: Claude Opus 5.5 --- src/main/java/com/iemr/flw/service/sync/StopTBSyncType.java | 4 +++- .../com/iemr/flw/service/sync/StopTBSyncCoverageTest.java | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/iemr/flw/service/sync/StopTBSyncType.java b/src/main/java/com/iemr/flw/service/sync/StopTBSyncType.java index d4c8308f8..00c993a73 100644 --- a/src/main/java/com/iemr/flw/service/sync/StopTBSyncType.java +++ b/src/main/java/com/iemr/flw/service/sync/StopTBSyncType.java @@ -45,7 +45,9 @@ public enum StopTBSyncType { GENERAL_OPD("generalOpd", "opd_visit", "SELECT v.BenVisitID FROM db_iemr.t_benvisitdetail v WHERE v.Deleted = 0 AND v.ProviderServiceMapID = :psm " + "AND v.BeneficiaryRegID IN (" + Scope.REG + ") AND v.BenVisitID > :after ORDER BY v.BenVisitID LIMIT :limit", - List.of("db_iemr.t_benvisitdetail", "db_iemr.t_benchiefcomplaint", "db_iemr.t_prescribeddrug")), + // t_prescription holds the visit's note (latest prescription instruction) + List.of("db_iemr.t_benvisitdetail", "db_iemr.t_benchiefcomplaint", "db_iemr.t_prescribeddrug", + "db_iemr.t_prescription")), /** Older free-text OPD rows; same record shape as GENERAL_OPD. */ GENERAL_OPD_LEGACY("generalOpdLegacy", "general_opd", diff --git a/src/test/java/com/iemr/flw/service/sync/StopTBSyncCoverageTest.java b/src/test/java/com/iemr/flw/service/sync/StopTBSyncCoverageTest.java index 161719b63..d7f08d734 100644 --- a/src/test/java/com/iemr/flw/service/sync/StopTBSyncCoverageTest.java +++ b/src/test/java/com/iemr/flw/service/sync/StopTBSyncCoverageTest.java @@ -23,7 +23,6 @@ class StopTBSyncCoverageTest { static final Map NOT_SYNCED_STOPTB = Map.ofEntries( Map.entry("db_iemr.t_phy_generalexam", "copy of the general examination for the standard tables; the app pulls tb_stoptb_general_examination"), Map.entry("db_iemr.t_benreferdetails", "copy written on save for the standard tables; not read by any STOP-TB pull"), - Map.entry("db_iemr.t_prescription", "OPD drugs are read from t_prescribeddrug, which is synced"), Map.entry("db_iemr.tb_stoptb_registration", "not read by any service"), Map.entry("db_iemr.tb_diagnostic_document", "the app does not pull documents"), Map.entry("db_iemr.tb_diagnostic_provider_token", "vendor credentials, server only"), From 891aeb6488be50defe31c15f65eee75818758290 Mon Sep 17 00:00:00 2001 From: Mithun James <1007084+drtechie@users.noreply.github.com> Date: Thu, 29 Oct 2026 11:56:42 +0530 Subject: [PATCH 04/12] fix(stoptb): bound beneficiaries per feed page and bind tokens to record types - A page carries at most 25 beneficiary records. Each costs several identity lookups and a FHIR call, so 200-500 per page could outlast the tablet's HTTP timeout. In a snapshot the token resumes after the last one sent; in a delta the row list stops before the one over the cap and the token resumes from the last row kept, so nothing is skipped. - Tokens carry the set of record types they were issued for. A request for a different set starts a new snapshot instead of missing the newly requested types' existing records. Co-Authored-By: Claude Opus 5.5 --- .../service/impl/StopTBSyncServiceImpl.java | 39 ++++++++++++++++--- .../com/iemr/flw/service/sync/SyncToken.java | 36 ++++++++++------- .../iemr/flw/service/sync/SyncTokenTest.java | 11 ++++-- 3 files changed, 64 insertions(+), 22 deletions(-) diff --git a/src/main/java/com/iemr/flw/service/impl/StopTBSyncServiceImpl.java b/src/main/java/com/iemr/flw/service/impl/StopTBSyncServiceImpl.java index 2d8bb51d5..c95484a2d 100644 --- a/src/main/java/com/iemr/flw/service/impl/StopTBSyncServiceImpl.java +++ b/src/main/java/com/iemr/flw/service/impl/StopTBSyncServiceImpl.java @@ -40,6 +40,11 @@ public class StopTBSyncServiceImpl implements StopTBSyncService { static final int DEFAULT_LIMIT = 200; static final int MAX_LIMIT = 500; + /** + * Beneficiary records are the expensive ones (several identity lookups and a FHIR call each), so a + * page carries at most this many; the token resumes from the last one sent. + */ + static final int MAX_BENEFICIARIES_PER_PAGE = 25; /** Lines younger than this may still have earlier seqs uncommitted; they wait for the next call. */ static final int SAFE_LAG_SECONDS = 5; /** The hub keeps 30 days of log; a token older than this may point at cleaned-up lines. */ @@ -90,9 +95,12 @@ public Map getChanges(String token, Integer providerServiceMapID && (parsed.kind() == SyncToken.Kind.SNAPSHOT || now - parsed.issuedAt() <= MAX_TOKEN_AGE_SECONDS); Scope scope = new Scope(providerServiceMapID, villageIds, requestedTypes(types)); + // A token covers only the types it was issued for; newly requested types need a snapshot. + usable = usable && parsed.typesMask() == scope.typesMask(); if (!usable) { if (parsed != null) logger.info("Sync token {} not usable here, starting a snapshot", parsed); - return snapshotPage(serverId, SyncToken.snapshot(serverId, safePoint(), 0, 0), scope, pageSize, authorisation); + return snapshotPage(serverId, SyncToken.snapshot(serverId, safePoint(), 0, 0, scope.typesMask()), + scope, pageSize, authorisation); } return parsed.kind() == SyncToken.Kind.SNAPSHOT ? snapshotPage(serverId, parsed, scope, pageSize, authorisation) @@ -116,6 +124,7 @@ private Map snapshotPage(String serverId, SyncToken at, Scope sc continue; } int want = pageSize - items.size(); + if (type == StopTBSyncType.BENEFICIARY) want = Math.min(want, MAX_BENEFICIARIES_PER_PAGE); List keys = snapshotKeys(type, scope, after, want); Map records = build(type, keys, authorisation); for (Long key : keys) { @@ -133,8 +142,8 @@ private Map snapshotPage(String serverId, SyncToken at, Scope sc boolean done = typeIndex >= types.length; SyncToken next = done - ? SyncToken.delta(serverId, at.seq(), System.currentTimeMillis() / 1000) - : SyncToken.snapshot(serverId, at.seq(), typeIndex, after); + ? SyncToken.delta(serverId, at.seq(), System.currentTimeMillis() / 1000, scope.typesMask()) + : SyncToken.snapshot(serverId, at.seq(), typeIndex, after, scope.typesMask()); return page(serverId, true, items, next, !done); } @@ -176,6 +185,19 @@ private Map deltaPage(String serverId, long from, Scope scope, i .setParameter("villages", scope.villages) .setParameter("limit", pageSize) .getResultList(); + boolean full = rows.size() == pageSize; + + // Keep beneficiary records per page bounded: stop before the one over the cap. The token then + // resumes from the last row kept, so nothing after it is skipped. + int beneficiaries = 0; + for (int i = 0; i < rows.size(); i++) { + if (!StopTBSyncType.BENEFICIARY.logEntity().equals(rows.get(i)[0])) continue; + if (++beneficiaries > MAX_BENEFICIARIES_PER_PAGE) { + rows = new ArrayList<>(rows.subList(0, i)); + full = true; + break; + } + } // Build each type's records in one batch, then emit in log order. Map> keysByType = new EnumMap<>(StopTBSyncType.class); @@ -205,9 +227,10 @@ private Map deltaPage(String serverId, long from, Scope scope, i } } - boolean hasMore = rows.size() == pageSize; + boolean hasMore = full; long nextSeq = hasMore ? lastSeq : safe; - return page(serverId, false, items, SyncToken.delta(serverId, nextSeq, System.currentTimeMillis() / 1000), hasMore); + return page(serverId, false, items, + SyncToken.delta(serverId, nextSeq, System.currentTimeMillis() / 1000, scope.typesMask()), hasMore); } // ── Records, in the same JSON each type's existing endpoint returns ─────── @@ -360,6 +383,12 @@ private static final class Scope { this.types = types; } + long typesMask() { + long mask = 0; + for (StopTBSyncType t : types) mask |= 1L << t.ordinal(); + return mask; + } + List logEntities() { List out = new ArrayList<>(); for (StopTBSyncType t : types) out.add(t.logEntity()); diff --git a/src/main/java/com/iemr/flw/service/sync/SyncToken.java b/src/main/java/com/iemr/flw/service/sync/SyncToken.java index d0f133b02..e64ac3888 100644 --- a/src/main/java/com/iemr/flw/service/sync/SyncToken.java +++ b/src/main/java/com/iemr/flw/service/sync/SyncToken.java @@ -3,14 +3,16 @@ /** * The tablet's opaque sync position. Two kinds: * - * d.<serverId>.<seq>.<issuedEpochSeconds> + * d.<serverId>.<seq>.<issuedEpochSeconds>.<typesMask> * changes after change-log line seq on this hub; - * s.<serverId>.<snapshotSeq>.<typeIndex>.<afterKey> + * s.<serverId>.<snapshotSeq>.<typeIndex>.<afterKey>.<typesMask> * part-way through a first download: the next page starts at type typeIndex, after afterKey, * and when it finishes the tablet continues with changes after snapshotSeq. * * serverId is the hub's MySQL server_uuid, so a token from another hub is never trusted. The issue * time lets the hub tell when the log lines a token points after may have been cleaned up. + * typesMask is the set of record types the token was issued for (bit = StopTBSyncType ordinal): a + * token only covers those types, so a request for a different set starts a new snapshot. * * @author Piramal Swasthya */ @@ -24,22 +26,25 @@ public enum Kind { DELTA, SNAPSHOT } private final long issuedAt; private final int typeIndex; private final long afterKey; + private final long typesMask; - private SyncToken(Kind kind, String serverId, long seq, long issuedAt, int typeIndex, long afterKey) { + private SyncToken(Kind kind, String serverId, long seq, long issuedAt, int typeIndex, long afterKey, + long typesMask) { this.kind = kind; this.serverId = serverId; this.seq = seq; this.issuedAt = issuedAt; this.typeIndex = typeIndex; this.afterKey = afterKey; + this.typesMask = typesMask; } - public static SyncToken delta(String serverId, long seq, long issuedAt) { - return new SyncToken(Kind.DELTA, serverId, seq, issuedAt, 0, 0); + public static SyncToken delta(String serverId, long seq, long issuedAt, long typesMask) { + return new SyncToken(Kind.DELTA, serverId, seq, issuedAt, 0, 0, typesMask); } - public static SyncToken snapshot(String serverId, long snapshotSeq, int typeIndex, long afterKey) { - return new SyncToken(Kind.SNAPSHOT, serverId, snapshotSeq, 0, typeIndex, afterKey); + public static SyncToken snapshot(String serverId, long snapshotSeq, int typeIndex, long afterKey, long typesMask) { + return new SyncToken(Kind.SNAPSHOT, serverId, snapshotSeq, 0, typeIndex, afterKey, typesMask); } /** null for a missing or unreadable token; the caller then starts a snapshot. */ @@ -47,11 +52,12 @@ public static SyncToken parse(String token) { if (token == null || token.isBlank()) return null; String[] p = token.trim().split("\\."); try { - if (p.length == 4 && "d".equals(p[0]) && !p[1].isEmpty()) { - return delta(p[1], Long.parseLong(p[2]), Long.parseLong(p[3])); + if (p.length == 5 && "d".equals(p[0]) && !p[1].isEmpty()) { + return delta(p[1], Long.parseLong(p[2]), Long.parseLong(p[3]), Long.parseLong(p[4])); } - if (p.length == 5 && "s".equals(p[0]) && !p[1].isEmpty()) { - return snapshot(p[1], Long.parseLong(p[2]), Integer.parseInt(p[3]), Long.parseLong(p[4])); + if (p.length == 6 && "s".equals(p[0]) && !p[1].isEmpty()) { + return snapshot(p[1], Long.parseLong(p[2]), Integer.parseInt(p[3]), Long.parseLong(p[4]), + Long.parseLong(p[5])); } } catch (NumberFormatException e) { return null; @@ -61,8 +67,8 @@ public static SyncToken parse(String token) { public String encode() { return kind == Kind.DELTA - ? "d." + serverId + "." + seq + "." + issuedAt - : "s." + serverId + "." + seq + "." + typeIndex + "." + afterKey; + ? "d." + serverId + "." + seq + "." + issuedAt + "." + typesMask + : "s." + serverId + "." + seq + "." + typeIndex + "." + afterKey + "." + typesMask; } public Kind kind() { @@ -90,6 +96,10 @@ public long afterKey() { return afterKey; } + public long typesMask() { + return typesMask; + } + @Override public String toString() { return encode(); diff --git a/src/test/java/com/iemr/flw/service/sync/SyncTokenTest.java b/src/test/java/com/iemr/flw/service/sync/SyncTokenTest.java index 968b21f03..bb72b31c7 100644 --- a/src/test/java/com/iemr/flw/service/sync/SyncTokenTest.java +++ b/src/test/java/com/iemr/flw/service/sync/SyncTokenTest.java @@ -10,20 +10,22 @@ class SyncTokenTest { @Test void deltaRoundTrips() { - SyncToken t = SyncToken.parse(SyncToken.delta(UUID, 1000, 1761700000L).encode()); + SyncToken t = SyncToken.parse(SyncToken.delta(UUID, 1000, 1761700000L, 0b101).encode()); assertEquals(SyncToken.Kind.DELTA, t.kind()); assertEquals(UUID, t.serverId()); assertEquals(1000, t.seq()); assertEquals(1761700000L, t.issuedAt()); + assertEquals(0b101, t.typesMask()); } @Test void snapshotRoundTrips() { - SyncToken t = SyncToken.parse(SyncToken.snapshot(UUID, 5230, 3, 900).encode()); + SyncToken t = SyncToken.parse(SyncToken.snapshot(UUID, 5230, 3, 900, 7).encode()); assertEquals(SyncToken.Kind.SNAPSHOT, t.kind()); assertEquals(5230, t.seq()); assertEquals(3, t.typeIndex()); assertEquals(900, t.afterKey()); + assertEquals(7, t.typesMask()); } @Test @@ -31,8 +33,9 @@ void unreadableTokensMeanStartOver() { assertNull(SyncToken.parse(null)); assertNull(SyncToken.parse("")); assertNull(SyncToken.parse("garbage")); - assertNull(SyncToken.parse("d." + UUID + ".x.1")); - assertNull(SyncToken.parse("d..1.1")); + assertNull(SyncToken.parse("d." + UUID + ".x.1.1")); + assertNull(SyncToken.parse("d." + UUID + ".1.1")); // no types: issued before types were bound + assertNull(SyncToken.parse("d..1.1.1")); assertNull(SyncToken.parse("q." + UUID + ".1.1")); } From ccb9d11cb47af293987300789e3d150c005de2fd Mon Sep 17 00:00:00 2001 From: Mithun James <1007084+drtechie@users.noreply.github.com> Date: Thu, 29 Oct 2026 12:43:08 +0530 Subject: [PATCH 05/12] docs(stoptb): the change log now comes from AMRIT-DB main migrations V132 / V32 Co-Authored-By: Claude Opus 5.5 --- .../java/com/iemr/flw/controller/StopTBSyncController.java | 2 +- src/main/java/com/iemr/flw/service/StopTBSyncService.java | 2 +- .../java/com/iemr/flw/service/impl/StopTBSyncServiceImpl.java | 4 ++-- src/main/java/com/iemr/flw/service/sync/StopTBSyncType.java | 2 +- .../com/iemr/flw/service/sync/StopTBSyncCoverageTest.java | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/iemr/flw/controller/StopTBSyncController.java b/src/main/java/com/iemr/flw/controller/StopTBSyncController.java index 9c1a8764f..c367b857c 100644 --- a/src/main/java/com/iemr/flw/controller/StopTBSyncController.java +++ b/src/main/java/com/iemr/flw/controller/StopTBSyncController.java @@ -29,7 +29,7 @@ public class StopTBSyncController { * Body: { token: string|null, providerServiceMapID: int, villageIds: [int], types: [string], limit: int }. * types: the record types the app applies (all when omitted); changing it requires a new snapshot. * Response: { supported, serverId, snapshot, items: [{type, op, key, version, data}], nextToken, hasMore }. - * Call again with nextToken while hasMore is true. On a server without the hub change log the + * Call again with nextToken while hasMore is true. On a server without the change log (AMRIT-DB before V132) the * response is { supported: false } and the app keeps using the getAll endpoints. */ @PostMapping("/changes") diff --git a/src/main/java/com/iemr/flw/service/StopTBSyncService.java b/src/main/java/com/iemr/flw/service/StopTBSyncService.java index 580542fe6..a4023f118 100644 --- a/src/main/java/com/iemr/flw/service/StopTBSyncService.java +++ b/src/main/java/com/iemr/flw/service/StopTBSyncService.java @@ -8,7 +8,7 @@ public interface StopTBSyncService { /** * One page of the STOP-TB changes feed for the given villages: a first full download when the * token is missing, from another hub or too old; otherwise only what changed after it. - * Returns {supported:false} where the hub change log isn't installed, so the app keeps its full pull. + * Returns {supported:false} where the change log isn't installed, so the app keeps its full pull. * types limits the feed to the record types the app applies (StopTBSyncType api names); null or * empty means all. The app must start a new snapshot (no token) whenever it changes this list. */ diff --git a/src/main/java/com/iemr/flw/service/impl/StopTBSyncServiceImpl.java b/src/main/java/com/iemr/flw/service/impl/StopTBSyncServiceImpl.java index c95484a2d..897e333a7 100644 --- a/src/main/java/com/iemr/flw/service/impl/StopTBSyncServiceImpl.java +++ b/src/main/java/com/iemr/flw/service/impl/StopTBSyncServiceImpl.java @@ -23,8 +23,8 @@ import java.util.*; /** - * STOP-TB changes feed, served only on hubs (camp laptops, and UAT central for testing), where - * AMRIT-DB's hub migrations install t_sync_change_log and its triggers. + * STOP-TB changes feed. AMRIT-DB V132 / V32 install t_sync_change_log and its triggers on every + * server; tablets call the feed on their camp hub (or UAT central when testing). * * The tablet keeps one opaque token. With none it gets a first full download in pages (snapshot); * after that, each call returns only the records with change-log lines after its token, in their diff --git a/src/main/java/com/iemr/flw/service/sync/StopTBSyncType.java b/src/main/java/com/iemr/flw/service/sync/StopTBSyncType.java index 00c993a73..c0e91e953 100644 --- a/src/main/java/com/iemr/flw/service/sync/StopTBSyncType.java +++ b/src/main/java/com/iemr/flw/service/sync/StopTBSyncType.java @@ -8,7 +8,7 @@ * * For each type: * - {@code logEntity}: the entity name the hub's change-log triggers write (AMRIT-DB - * db/migration/dbiemr_hub and dbidentity_hub); + * dbiemr/V132__Sync_Change_Log.sql and dbidentity/V32__Sync_Change_Log_Beneficiary.sql); * - {@code snapshotKeySql}: the keys in scope, for the first full download, with the same filters * the type's getAll endpoint applies. Parameters: :psm, :villages, :after, :limit; * - {@code sourceTables}: every table the type's record is built from. Each must have the change-log diff --git a/src/test/java/com/iemr/flw/service/sync/StopTBSyncCoverageTest.java b/src/test/java/com/iemr/flw/service/sync/StopTBSyncCoverageTest.java index d7f08d734..dea113d81 100644 --- a/src/test/java/com/iemr/flw/service/sync/StopTBSyncCoverageTest.java +++ b/src/test/java/com/iemr/flw/service/sync/StopTBSyncCoverageTest.java @@ -12,7 +12,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; /** - * Every table FLW-API maps must be either synced to STOP-TB tablets through the hub change log + * Every table FLW-API maps must be either synced to STOP-TB tablets through the change log * (listed in a {@link StopTBSyncType}'s sourceTables, which the hub checks for triggers) or listed * below as deliberately not synced. A new entity fails this test until someone decides which, so a * STOP-TB table can't be added without its change-log triggers. @@ -76,7 +76,7 @@ void everyMappedTableIsSyncedOrDeliberatelyNot() throws Exception { Collections.sort(unclassified); assertTrue(unclassified.isEmpty(), "Tables neither synced to STOP-TB tablets nor listed as not synced: " + unclassified + ". If the STOP-TB app reads it, add it to a StopTBSyncType's sourceTables and add " - + "change-log triggers in AMRIT-DB db/migration/*_hub; otherwise list it here with a reason."); + + "change-log triggers in a new AMRIT-DB migration; otherwise list it here with a reason."); } @Test From cd298ff8059b405302c9eb162ff5e794cd19e5ec Mon Sep 17 00:00:00 2001 From: Vishwanath Balkur <118195001+vishwab1@users.noreply.github.com> Date: Thu, 29 Oct 2026 12:48:31 +0530 Subject: [PATCH 06/12] Efficency on Stoptbapp (#394) * perf(worklist): batch per-page lookups in getBeneficiaryData Load mapping, details, account, image, address, contact, beneficiary ID, rmnch, born-birth, household, flow, anthropometry, vitals and user lookups once per page instead of once per beneficiary. Read ABHA from m_benhealthidmapping/t_healthid directly instead of calling FHIR-API per beneficiary. Falls back to the existing per-row queries if the batch load fails or a single-result lookup finds more than one row. Co-Authored-By: Claude Opus 5.5 * perf(worklist): page size 50 and per-page timing log door-to-door-page-size 10 -> 50 so a village pull needs 5x fewer requests. Log one [WORKLIST-TIMING] line per Stop TB village worklist page with the time spent in count, ids, address, prefetch, build and JSON. Co-Authored-By: Claude Opus 5.5 * perf(worklist): index-only count and page query; per-request timing log Count village flow rows without the mapping join, and page with a deferred join so OFFSET skips on idx_flow_psm_village_reg only and just the page's rows are joined to the mapping. Ties on registrationDate are ordered by ben_flow_id so a page boundary cannot skip or repeat a beneficiary. An in-range page with no mapped rows returns an empty page instead of "No record found". Add ApiTimingFilter: one [API-TIMING] line per request with method, path, status and time. Co-Authored-By: Claude Opus 5.5 * perf(stoptb): faster TB suspected and Stop TB getAll responses TB suspected getAll looked up the full user detail (role + facility queries) for every confirmed row only to read the state for the incentive check; look it up once per user. The incentive rules and calls are unchanged. Stop TB general exam, screening, OPD and diagnostics getAll built the JSON text, parsed it back and wrote it again; build the tree once via OutputResponse.setResponse(JsonElement). Load those rows read-only. Responses are byte-identical to before (Stop TB getAlls and the FLW app worklist/suspected paths compared on the 3307 test DB). Co-Authored-By: Claude Opus 5.5 * perf(stoptb): one-query diagnostic status summary; leaner suspected and OPD lists Diagnostic status summary read each beneficiary's latest order five times, once per status bucket; read beneficiaryId + status once and bucket in Java with the same status groups (case-insensitive, like the database comparison). Bucket contents are unchanged; order inside a bucket was never fixed by the query. TB suspected getAll copies the DTO fields directly instead of ModelMapper per row, and its village subquery hints idx_flow_psm_village_reg (MySQL ignores the hint where that index is absent). Standard OPD records for a village start from the village's beneficiaries instead of every visit of the PSM; the same visits match. Co-Authored-By: Claude Opus 5.5 * perf(stoptb): keep TB suspected speed-up on the Stop TB village path only Restore IncentiveLogicService/IncentiveLogicImpl to their original code and run the FLW (ashaId) path of TB suspected getAll exactly as before. The Stop TB village path reads each user's state once and calls the unchanged incentiveForTbSuspected only for CG users, the only state where it acts. Co-Authored-By: Claude Opus 5.5 * perf(worklist): restore original village count and page queries The index-only count also counted flow rows that have no i_beneficiarymapping row (423 on the reference DB, mostly PSM 1725), so villages with such rows got extra or empty pages and a fully unmapped village got an empty success instead of "No record found". Put countVillageWorklist, getVillageWorklistAddressIds and the empty-page handling back to the release-3.12.0 behaviour. The batch loader, page size and timing logs stay. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .../iemr/flw/controller/StopTBController.java | 8 +- .../flw/repo/identity/BeneficiaryRepo.java | 7 + .../flw/repo/iemr/DiagnosticOrderRepo.java | 12 ++ .../flw/repo/iemr/StopTBDiagnosticsRepo.java | 4 + .../iemr/StopTBGeneralExaminationRepo.java | 4 + .../flw/repo/iemr/StopTBGeneralOpdRepo.java | 4 + .../iemr/flw/repo/iemr/TBScreeningRepo.java | 4 + .../iemr/flw/repo/iemr/TBSuspectedRepo.java | 7 +- .../service/impl/BeneficiaryServiceImpl.java | 154 ++++++++++---- .../impl/DiagnosticOrderServiceImpl.java | 30 ++- .../service/impl/TBSuspectedServiceImpl.java | 108 ++++++++-- .../service/impl/WorklistPagePrefetch.java | 191 ++++++++++++++++++ .../com/iemr/flw/utils/ApiTimingFilter.java | 40 ++++ .../flw/utils/response/OutputResponse.java | 9 + src/main/resources/application.properties | 2 +- 15 files changed, 519 insertions(+), 65 deletions(-) create mode 100644 src/main/java/com/iemr/flw/service/impl/WorklistPagePrefetch.java create mode 100644 src/main/java/com/iemr/flw/utils/ApiTimingFilter.java diff --git a/src/main/java/com/iemr/flw/controller/StopTBController.java b/src/main/java/com/iemr/flw/controller/StopTBController.java index 49cb691b8..e9a38513c 100644 --- a/src/main/java/com/iemr/flw/controller/StopTBController.java +++ b/src/main/java/com/iemr/flw/controller/StopTBController.java @@ -52,7 +52,7 @@ public String getAllGeneralExaminations(@RequestBody Map body) { if (raw == null) throw new Exception("providerServiceMapID is required"); Integer villageID = body.get("villageID") != null ? Integer.parseInt(body.get("villageID").toString()) : null; Map data = stopTBService.getAllGeneralExaminations(Integer.parseInt(raw.toString()), villageID); - response.setResponse(gson.toJson(data)); + response.setResponse(gson.toJsonTree(data)); } catch (Exception e) { logger.error("Error in getAllGeneralExaminations: " + e); response.setError(5000, "Error fetching general examinations: " + e.getMessage()); @@ -85,7 +85,7 @@ public String getAllNurseTBScreenings(@RequestBody Map body) { if (raw == null) throw new Exception("providerServiceMapID is required"); Integer villageID = body.get("villageID") != null ? Integer.parseInt(body.get("villageID").toString()) : null; Map data = stopTBService.getAllNurseTBScreenings(Integer.parseInt(raw.toString()), villageID); - response.setResponse(gson.toJson(data)); + response.setResponse(gson.toJsonTree(data)); } catch (Exception e) { logger.error("Error in getAllNurseTBScreenings: " + e); response.setError(5000, "Error fetching TB screenings: " + e.getMessage()); @@ -122,7 +122,7 @@ public String getAllGeneralOpd(@RequestBody Map body) { if (raw == null) throw new Exception("providerServiceMapID is required"); Integer villageID = body.get("villageID") != null ? Integer.parseInt(body.get("villageID").toString()) : null; Map data = stopTBService.getAllGeneralOpd(Integer.parseInt(raw.toString()), villageID); - response.setResponse(gson.toJson(data)); + response.setResponse(gson.toJsonTree(data)); } catch (Exception e) { logger.error("Error in getAllGeneralOpd: " + e); response.setError(5000, "Error fetching general OPD records: " + e.getMessage()); @@ -155,7 +155,7 @@ public String getAllDiagnostics(@RequestBody Map body) { if (raw == null) throw new Exception("providerServiceMapID is required"); Integer villageID = body.get("villageID") != null ? Integer.parseInt(body.get("villageID").toString()) : null; Map data = stopTBService.getAllDiagnostics(Integer.parseInt(raw.toString()), villageID); - response.setResponse(gson.toJson(data)); + response.setResponse(gson.toJsonTree(data)); } catch (Exception e) { logger.error("Error in getAllDiagnostics: " + e); response.setError(5000, "Error fetching diagnostics records: " + e.getMessage()); diff --git a/src/main/java/com/iemr/flw/repo/identity/BeneficiaryRepo.java b/src/main/java/com/iemr/flw/repo/identity/BeneficiaryRepo.java index a1b0b9333..cd7316c80 100644 --- a/src/main/java/com/iemr/flw/repo/identity/BeneficiaryRepo.java +++ b/src/main/java/com/iemr/flw/repo/identity/BeneficiaryRepo.java @@ -107,6 +107,13 @@ Page getBenDataWithinDates( @Query(nativeQuery = true, value = " SELECT HealthID,HealthIdNumber,isNewAbha FROM db_iemr.t_healthid WHERE HealthIdNumber = :healthIdNumber ") ArrayList getBenHealthDetails(@Param("healthIdNumber") String healthIdNumber); + // Same rows FHIR-API getBenhealthID reads, so the worklist can fill healthId/healthIdNumber without an HTTP call + @Query(nativeQuery = true, value = " SELECT HealthID, HealthIdNumber FROM db_iemr.m_benhealthidmapping WHERE BeneficiaryRegID = :benRegId ") + List getHealthIdMappingRows(@Param("benRegId") Long benRegId); + + @Query(nativeQuery = true, value = " SELECT HealthId FROM db_iemr.t_healthid WHERE HealthIdNumber = :healthIdNumber ") + List getHealthIdByNumber(@Param("healthIdNumber") String healthIdNumber); + // benRegId has no unique constraint on i_beneficiarymapping — a beneficiary can have // more than one mapping row, so this must return a list, not assume a single result. @Query("SELECT b FROM RMNCHMBeneficiarymapping b WHERE b.benRegId = :benRegId ORDER BY b.benMapId DESC") diff --git a/src/main/java/com/iemr/flw/repo/iemr/DiagnosticOrderRepo.java b/src/main/java/com/iemr/flw/repo/iemr/DiagnosticOrderRepo.java index 86e811c81..729e90f5a 100644 --- a/src/main/java/com/iemr/flw/repo/iemr/DiagnosticOrderRepo.java +++ b/src/main/java/com/iemr/flw/repo/iemr/DiagnosticOrderRepo.java @@ -103,4 +103,16 @@ List findBeneficiaryIdsClosed(@Param("orderType") String orderType, "AND (:providerServiceMapId IS NULL OR b.providerServiceMapId = :providerServiceMapId))") List findBeneficiaryIdsAwaitingManualEntry(@Param("orderType") String orderType, @Param("villageId") Integer villageId, @Param("providerServiceMapId") Integer providerServiceMapId); + + // The five queries above in one pass: beneficiaryId + status of each beneficiary's latest order, + // same filters, for the caller to bucket by status. + @Query("SELECT o.beneficiaryId, o.status FROM DiagnosticOrder o WHERE o.orderType = :orderType AND o.deleted = false " + + "AND o.status IN ('PENDING', 'IN_PROGRESS', 'COMPLETED', 'FAILED', 'CLOSED', 'CANCELLED', 'EXPIRED', 'REFUSED', 'MANUAL_ENTRY') " + + "AND o.id = (SELECT MAX(o2.id) FROM DiagnosticOrder o2 " + + "WHERE o2.beneficiaryId = o.beneficiaryId AND o2.orderType = :orderType AND o2.deleted = false) " + + "AND o.beneficiaryId IN (SELECT b.beneficiaryID FROM BenFlowStatus b WHERE b.deleted = false " + + "AND (:villageId IS NULL OR b.villageID = :villageId) " + + "AND (:providerServiceMapId IS NULL OR b.providerServiceMapId = :providerServiceMapId))") + List findLatestOrderStatusByBeneficiary(@Param("orderType") String orderType, + @Param("villageId") Integer villageId, @Param("providerServiceMapId") Integer providerServiceMapId); } diff --git a/src/main/java/com/iemr/flw/repo/iemr/StopTBDiagnosticsRepo.java b/src/main/java/com/iemr/flw/repo/iemr/StopTBDiagnosticsRepo.java index de3fa9096..19bb9acfa 100644 --- a/src/main/java/com/iemr/flw/repo/iemr/StopTBDiagnosticsRepo.java +++ b/src/main/java/com/iemr/flw/repo/iemr/StopTBDiagnosticsRepo.java @@ -4,6 +4,8 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; +import org.springframework.data.jpa.repository.QueryHints; +import jakarta.persistence.QueryHint; import org.springframework.data.repository.query.Param; import org.springframework.stereotype.Repository; import org.springframework.transaction.annotation.Transactional; @@ -19,6 +21,8 @@ public interface StopTBDiagnosticsRepo extends JpaRepository findAllByProviderServiceMapID(@Param("psmId") Integer psmId); + // Read-only: getAll only reads these rows, so Hibernate keeps no change-tracking snapshot per row + @QueryHints(@QueryHint(name = "org.hibernate.readOnly", value = "true")) @Query("SELECT d FROM StopTBDiagnostics d WHERE d.benRegID IN " + "(SELECT b.beneficiaryRegID FROM BenFlowStatus b WHERE b.providerServiceMapId = :psmId AND b.villageID = :villageId) " + "AND d.deleted = false ORDER BY d.visitDate DESC") diff --git a/src/main/java/com/iemr/flw/repo/iemr/StopTBGeneralExaminationRepo.java b/src/main/java/com/iemr/flw/repo/iemr/StopTBGeneralExaminationRepo.java index 944e99a65..798fcaf85 100644 --- a/src/main/java/com/iemr/flw/repo/iemr/StopTBGeneralExaminationRepo.java +++ b/src/main/java/com/iemr/flw/repo/iemr/StopTBGeneralExaminationRepo.java @@ -5,6 +5,8 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; +import org.springframework.data.jpa.repository.QueryHints; +import jakarta.persistence.QueryHint; import org.springframework.data.repository.query.Param; import org.springframework.stereotype.Repository; import org.springframework.transaction.annotation.Transactional; @@ -32,6 +34,8 @@ StopTBGeneralExamination findByBeneficiaryRegIDAndVisitCode(@Param("beneficiaryR @Query("SELECT e FROM StopTBGeneralExamination e WHERE e.providerServiceMapID = :psmId AND e.deleted = false ORDER BY e.createdDate DESC") List findAllByProviderServiceMapID(@Param("psmId") Integer psmId); + // Read-only: getAll only reads these rows, so Hibernate keeps no change-tracking snapshot per row + @QueryHints(@QueryHint(name = "org.hibernate.readOnly", value = "true")) @Query("SELECT e FROM StopTBGeneralExamination e WHERE e.beneficiaryRegID IN " + "(SELECT b.beneficiaryRegID FROM BenFlowStatus b WHERE b.providerServiceMapId = :psmId AND b.villageID = :villageId) " + "AND e.deleted = false ORDER BY e.createdDate DESC") diff --git a/src/main/java/com/iemr/flw/repo/iemr/StopTBGeneralOpdRepo.java b/src/main/java/com/iemr/flw/repo/iemr/StopTBGeneralOpdRepo.java index 3cb8868f8..d35925a26 100644 --- a/src/main/java/com/iemr/flw/repo/iemr/StopTBGeneralOpdRepo.java +++ b/src/main/java/com/iemr/flw/repo/iemr/StopTBGeneralOpdRepo.java @@ -5,6 +5,8 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; +import org.springframework.data.jpa.repository.QueryHints; +import jakarta.persistence.QueryHint; import org.springframework.data.repository.query.Param; import org.springframework.stereotype.Repository; import org.springframework.transaction.annotation.Transactional; @@ -30,6 +32,8 @@ public interface StopTBGeneralOpdRepo extends JpaRepository findAllByProviderServiceMapID(@Param("psmId") Integer psmId); + // Read-only: getAll only reads these rows, so Hibernate keeps no change-tracking snapshot per row + @QueryHints(@QueryHint(name = "org.hibernate.readOnly", value = "true")) @Query("SELECT o FROM StopTBGeneralOpd o WHERE o.benRegID IN " + "(SELECT b.beneficiaryRegID FROM BenFlowStatus b WHERE b.providerServiceMapId = :psmId AND b.villageID = :villageId) " + "AND o.deleted = false ORDER BY o.createdDate DESC") diff --git a/src/main/java/com/iemr/flw/repo/iemr/TBScreeningRepo.java b/src/main/java/com/iemr/flw/repo/iemr/TBScreeningRepo.java index f656dc91d..cfb0665db 100644 --- a/src/main/java/com/iemr/flw/repo/iemr/TBScreeningRepo.java +++ b/src/main/java/com/iemr/flw/repo/iemr/TBScreeningRepo.java @@ -5,6 +5,8 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; +import org.springframework.data.jpa.repository.QueryHints; +import jakarta.persistence.QueryHint; import org.springframework.data.repository.query.Param; import org.springframework.stereotype.Repository; import org.springframework.transaction.annotation.Transactional; @@ -37,6 +39,8 @@ public interface TBScreeningRepo extends JpaRepository { @Query("SELECT tbs FROM TBScreening tbs WHERE tbs.providerServiceMapID = :psmId AND tbs.deleted = false ORDER BY tbs.visitDate DESC") List findAllByProviderServiceMapID(@Param("psmId") Integer psmId); + // Read-only: getAll only reads these rows, so Hibernate keeps no change-tracking snapshot per row + @QueryHints(@QueryHint(name = "org.hibernate.readOnly", value = "true")) @Query("SELECT tbs FROM TBScreening tbs WHERE tbs.benRegID IN " + "(SELECT b.beneficiaryRegID FROM BenFlowStatus b WHERE b.providerServiceMapId = :psmId AND b.villageID = :villageId) " + "AND tbs.deleted = false ORDER BY tbs.visitDate DESC") diff --git a/src/main/java/com/iemr/flw/repo/iemr/TBSuspectedRepo.java b/src/main/java/com/iemr/flw/repo/iemr/TBSuspectedRepo.java index 3df494fa4..277d21fd6 100644 --- a/src/main/java/com/iemr/flw/repo/iemr/TBSuspectedRepo.java +++ b/src/main/java/com/iemr/flw/repo/iemr/TBSuspectedRepo.java @@ -29,8 +29,11 @@ TBSuspected getByUserIdAndBenIdAndVisitCode(@Param("benId") Long benId, @Param(" @Query("SELECT tbs FROM TBSuspected tbs WHERE tbs.userId = :userId and tbs.visitDate >= :fromDate and tbs.visitDate <= :toDate") List getByUserId(@Param("userId") Integer userId, @Param("fromDate") Timestamp fromDate, @Param("toDate") Timestamp toDate); - @Query("SELECT tbs FROM TBSuspected tbs WHERE tbs.benId IN " + - "(SELECT b.beneficiaryID FROM BenFlowStatus b WHERE b.providerServiceMapId = :psmId AND b.villageID = :villageId)") + // Same query as before in SQL, with an optimizer hint so the village subquery reads + // idx_flow_psm_village_reg instead of the PSM-only index (ignored if that index is missing). + @Query(nativeQuery = true, value = "SELECT t.* FROM db_iemr.tb_suspected t WHERE t.ben_id IN " + + "(SELECT /*+ INDEX(b idx_flow_psm_village_reg) */ b.beneficiary_id FROM db_iemr.i_ben_flow_outreach b " + + "WHERE b.providerServiceMapID = :psmId AND b.villageID = :villageId)") List getByProviderServiceMapIdAndVillageId(@Param("psmId") Integer psmId, @Param("villageId") Integer villageId); @Transactional diff --git a/src/main/java/com/iemr/flw/service/impl/BeneficiaryServiceImpl.java b/src/main/java/com/iemr/flw/service/impl/BeneficiaryServiceImpl.java index 8bda704c8..bc1edc372 100644 --- a/src/main/java/com/iemr/flw/service/impl/BeneficiaryServiceImpl.java +++ b/src/main/java/com/iemr/flw/service/impl/BeneficiaryServiceImpl.java @@ -120,6 +120,9 @@ public class BeneficiaryServiceImpl implements BeneficiaryService { @Autowired private BenPhysicalVitalRepo benPhysicalVitalRepo; + @Autowired + private WorklistPagePrefetch worklistPagePrefetch; + private static final DateTimeFormatter FORMATTER = DateTimeFormatter.ofPattern("dd-MM-yyyy"); @@ -142,8 +145,10 @@ public String getBenData(GetBenRequestHandler request, String authorisation) thr // O(village size), on every page including page 0. See BeneficiaryRepo for the // underlying query. if (request.getProviderServiceMapID() != null && request.getVillageID() != null) { + long tStart = System.currentTimeMillis(); long totalCount = beneficiaryRepo.countVillageWorklist( request.getProviderServiceMapID(), request.getVillageID()); + long tCount = System.currentTimeMillis(); if (totalCount == 0) return null; @@ -153,6 +158,7 @@ public String getBenData(GetBenRequestHandler request, String authorisation) thr List addressIds = beneficiaryRepo.getVillageWorklistAddressIds( request.getProviderServiceMapID(), request.getVillageID(), pageSize, offset); + long tIds = System.currentTimeMillis(); if (addressIds.isEmpty()) return null; @@ -160,17 +166,28 @@ public String getBenData(GetBenRequestHandler request, String authorisation) thr // Hibernate 6.4.1 bug there: "Binding is multi-valued; illegal call to // #getBindValue" even with a well-formed, non-empty id list (confirmed live // against 192.168.45.40 - the 10 rows returned have no NULLs/type issues, so - // this isn't a data problem). Looping getAddressById() per id sidesteps - // Hibernate's collection-parameter binding entirely - only ever pageSize (10) - // calls per page, same order-preservation as before. - List pageAddresses = addressIds.stream() - .map(beneficiaryRepo::getAddressById) - .filter(java.util.Objects::nonNull) - .collect(Collectors.toList()); + // this isn't a data problem). WorklistPagePrefetch reads them in one native query + // with the ids as numeric literals (no collection binding), keeping page order; + // the per-id loop stays as the fallback. + List pageAddresses; + try { + pageAddresses = worklistPagePrefetch.addressesInOrder(addressIds); + } catch (Exception e) { + logger.warn("Worklist address batch failed, reading one by one: " + e.getMessage()); + pageAddresses = addressIds.stream() + .map(beneficiaryRepo::getAddressById) + .filter(java.util.Objects::nonNull) + .collect(Collectors.toList()); + } if (pageAddresses.isEmpty()) return null; + long tAddr = System.currentTimeMillis(); - return getMappingsForAddressIDs(pageAddresses, totalPage, authorisation); + String timingLabel = "[WORKLIST-TIMING] psm=" + request.getProviderServiceMapID() + + " village=" + request.getVillageID() + " page=" + request.getPageNo() + "/" + totalPage + + " pageSize=" + pageSize + " villageTotal=" + totalCount + + " | countMs=" + (tCount - tStart) + " idsMs=" + (tIds - tCount) + " addressMs=" + (tAddr - tIds); + return getMappingsForAddressIDs(pageAddresses, totalPage, authorisation, timingLabel, tStart); } // Normal FLW/ASHA path @@ -214,21 +231,35 @@ public String getBenData(GetBenRequestHandler request, String authorisation) thr return getMappingsForAddressIDs( pageResult.getContent(), pageResult.getTotalPages(), - authorisation + authorisation, + null, + 0L ); } + // timingLabel != null logs one [WORKLIST-TIMING] line per page (Stop TB village path) so camp-laptop + // logs show where each page's time goes. private String getMappingsForAddressIDs(List addressList, int totalPage, - String authorisation) { + String authorisation, String timingLabel, long tStart) { + long tBuildStart = System.currentTimeMillis(); List> resultList = buildBeneficiaryRecords(addressList, authorisation); + long tBuild = System.currentTimeMillis(); Map response = new HashMap<>(); response.put("data", resultList); response.put("pageSize", Integer.parseInt(door_to_door_page_size)); response.put("totalPage", totalPage); Gson gson = new GsonBuilder().setDateFormat("MMM dd, yyyy h:mm:ss a").create(); - return gson.toJson(response); + String json = gson.toJson(response); + + if (timingLabel != null) { + long tEnd = System.currentTimeMillis(); + logger.info(timingLabel + " buildMs=" + (tBuild - tBuildStart) + " jsonMs=" + (tEnd - tBuild) + + " totalMs=" + (tEnd - tStart) + " rows=" + resultList.size() + "/" + addressList.size() + + " bytes=" + json.length()); + } + return json; } // One record per beneficiary, in the same JSON shape getBeneficiaryData returns, keyed by BenRegId. @@ -266,10 +297,20 @@ private List> buildBeneficiaryRecords(List resultMap; ArrayList> resultList = new ArrayList<>(); + // One query per table for the whole page; if it fails, every lookup below falls back to its + // original per-beneficiary query. + WorklistPagePrefetch.Page pf = null; + try { + pf = worklistPagePrefetch.load(addressList); + } catch (Exception e) { + logger.warn("Worklist page prefetch failed, using per-beneficiary queries: " + e.getMessage()); + } + for (RMNCHMBeneficiaryaddress a : addressList) { // exception by-passing try { - List mappingsForAddress = beneficiaryRepo.getByAddressID(a.getId()); + List mappingsForAddress = pf != null ? pf.mappings(a.getId()) + : beneficiaryRepo.getByAddressID(a.getId()); if(!mappingsForAddress.isEmpty()){ RMNCHMBeneficiarymapping m = mappingsForAddress.get(0); if (m != null) { @@ -282,38 +323,53 @@ private List> buildBeneficiaryRecords(List healthDetails = getBenHealthDetails(m.getBenRegId()); + Long regIdLong = m.getBenRegId() != null ? m.getBenRegId().longValue() : null; + // no m_benhealthidmapping row -> the original lookup returns an empty map too + Map healthDetails = (pf != null && regIdLong != null && !pf.hasHealthId(regIdLong)) + ? new HashMap<>() : getBenHealthDetails(m.getBenRegId()); if (m.getBenDetailsId() != null) { - benDetailsOBJ = beneficiaryRepo.getDetailsById(m.getBenDetailsId()); + benDetailsOBJ = pf != null ? pf.detail(m.getBenDetailsId()) + : beneficiaryRepo.getDetailsById(m.getBenDetailsId()); } if (m.getBenAccountID() != null) { - benAccountOBJ = beneficiaryRepo.getAccountById(m.getBenAccountID()); + benAccountOBJ = pf != null ? pf.account(m.getBenAccountID()) + : beneficiaryRepo.getAccountById(m.getBenAccountID()); } if (m.getBenImageId() != null) { - benImageOBJ = beneficiaryRepo.getImageById(m.getBenImageId().longValue()); + benImageOBJ = pf != null ? pf.image(m.getBenImageId()) + : beneficiaryRepo.getImageById(m.getBenImageId().longValue()); } if (m.getBenAddressId() != null) { - benAddressOBJ = beneficiaryRepo.getAddressById(m.getBenAddressId()); + benAddressOBJ = pf != null ? pf.address(m.getBenAddressId()) + : beneficiaryRepo.getAddressById(m.getBenAddressId()); } if (m.getBenContactsId() != null) { - benContactOBJ = beneficiaryRepo.getContactById(m.getBenContactsId()); + benContactOBJ = pf != null ? pf.contact(m.getBenContactsId()) + : beneficiaryRepo.getContactById(m.getBenContactsId()); } BigInteger benID = null; - if (m.getBenRegId() != null) - benID = beneficiaryRepo.getBenIdFromRegID(m.getBenRegId().longValue()); + if (regIdLong != null) { + // more than one row: let the original single-result query decide, as before + benID = (pf != null && pf.benIds(regIdLong).size() <= 1) + ? pf.benIds(regIdLong).stream().findFirst().orElse(null) + : beneficiaryRepo.getBenIdFromRegID(regIdLong); + } - if (m.getBenRegId() != null) { - List detailsByRegID = beneficiaryRepo - .getDetailsByRegID((m.getBenRegId()).longValue()); + if (regIdLong != null) { + List detailsByRegID = pf != null ? pf.rmnch(regIdLong) + : beneficiaryRepo.getDetailsByRegID(regIdLong); if(!detailsByRegID.isEmpty()){ benDetailsRMNCH_OBJ = detailsByRegID.get(0); } - benBotnBirthRMNCH_ROBJ = beneficiaryRepo.getBornBirthByRegID((m.getBenRegId()).longValue()); + benBotnBirthRMNCH_ROBJ = (pf != null && pf.bornBirth(regIdLong).size() <= 1) + ? pf.bornBirth(regIdLong).stream().findFirst().orElse(null) + : beneficiaryRepo.getBornBirthByRegID(regIdLong); if (benDetailsRMNCH_OBJ != null && benDetailsRMNCH_OBJ.getHouseoldId() != null) { - List householdsById = houseHoldRepo - .getByHouseHoldID(benDetailsRMNCH_OBJ.getHouseoldId()); + List householdsById = pf != null + ? pf.households(benDetailsRMNCH_OBJ.getHouseoldId()) + : houseHoldRepo.getByHouseHoldID(benDetailsRMNCH_OBJ.getHouseoldId()); if(!householdsById.isEmpty()){ benHouseHoldRMNCH_ROBJ = householdsById.get(0); } @@ -425,7 +481,8 @@ private List> buildBeneficiaryRecords(List flows = benFlowStatusRepo.findByBeneficiaryRegID(m.getBenRegId().longValue()); + List flows = pf != null ? pf.flows(regIdLong) + : benFlowStatusRepo.findByBeneficiaryRegID(m.getBenRegId().longValue()); if (!flows.isEmpty() && flows.get(0).getDob() != null) benDetailsRMNCH_OBJ.setDob(flows.get(0).getDob()); } @@ -568,10 +625,10 @@ private List> buildBeneficiaryRecords(List anthroList = - benAnthropometryRepo.findByBeneficiaryRegIDOrderByCreatedDateDesc(benRegIdLong); - List vitalList = - benPhysicalVitalRepo.findByBeneficiaryRegIDOrderByCreatedDateDesc(benRegIdLong); + List anthroList = pf != null ? pf.anthropometry(benRegIdLong) + : benAnthropometryRepo.findByBeneficiaryRegIDOrderByCreatedDateDesc(benRegIdLong); + List vitalList = pf != null ? pf.vitals(benRegIdLong) + : benPhysicalVitalRepo.findByBeneficiaryRegIDOrderByCreatedDateDesc(benRegIdLong); Map anthropometry = new HashMap<>(); for (BenAnthropometryDetail anthro : anthroList) { @@ -627,13 +684,24 @@ private List> buildBeneficiaryRecords(List 0) resultMap.put("ashaId", userID); } - // get HealthID of ben - if (m.getBenRegId() != null) { - fetchHealthIdByBenRegID(m.getBenRegId().longValue(), authorisation, resultMap); + // get HealthID of ben: read locally (same table FHIR-API reads), only when a mapping exists + if (regIdLong != null) { + if (pf != null) { + if (pf.hasHealthId(regIdLong)) putLocalHealthId(regIdLong, resultMap); + } else { + fetchHealthIdByBenRegID(regIdLong, authorisation, resultMap); + } } resultList.add(resultMap); @@ -697,6 +765,22 @@ private Map getBenBenVisitDetails(BigInteger benRegId) { return healthDetails; } + // Mirrors FHIR-API HealthIDServiceImpl.getBenHealthID: every m_benhealthidmapping row, with healthId + // replaced by t_healthid.HealthId when that is filled; the last row wins, as in fetchHealthIdByBenRegID. + private void putLocalHealthId(Long benRegID, Map resultMap) { + for (Object[] row : beneficiaryRepo.getHealthIdMappingRows(benRegID)) { + String healthId = row[0] != null ? row[0].toString() : null; + String healthIdNumber = row[1] != null ? row[1].toString() : null; + if (healthIdNumber != null) { + List fromHealthIdTable = beneficiaryRepo.getHealthIdByNumber(healthIdNumber); + if (!fromHealthIdTable.isEmpty() && fromHealthIdTable.get(0) != null && !fromHealthIdTable.get(0).isEmpty()) + healthId = fromHealthIdTable.get(0); + } + if (healthId != null) resultMap.put("healthId", healthId); + if (healthIdNumber != null) resultMap.put("healthIdNumber", healthIdNumber); + } + } + public void fetchHealthIdByBenRegID(Long benRegID, String authorization, Map resultMap) { Map requestMap = new HashMap(); requestMap.put("beneficiaryRegID", benRegID); diff --git a/src/main/java/com/iemr/flw/service/impl/DiagnosticOrderServiceImpl.java b/src/main/java/com/iemr/flw/service/impl/DiagnosticOrderServiceImpl.java index 19a00862d..3bc5c5a24 100644 --- a/src/main/java/com/iemr/flw/service/impl/DiagnosticOrderServiceImpl.java +++ b/src/main/java/com/iemr/flw/service/impl/DiagnosticOrderServiceImpl.java @@ -38,6 +38,7 @@ import org.springframework.stereotype.Service; import java.sql.Timestamp; +import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.Set; @@ -814,16 +815,25 @@ private DiagnosticOrderResultDto toResultDto(DiagnosticOrder order) { public DiagnosticOrderStatusSummaryDto getOrderStatusSummary(String orderType, Integer villageId, Integer providerServiceMapId) { DiagnosticOrderType type = DiagnosticOrderType.fromCode(orderType); - List awaitingProviderResult = diagnosticOrderRepo - .findBeneficiaryIdsAwaitingProviderResult(type.name(), villageId, providerServiceMapId); - List completed = diagnosticOrderRepo - .findBeneficiaryIdsCompleted(type.name(), villageId, providerServiceMapId); - List failed = diagnosticOrderRepo - .findBeneficiaryIdsFailed(type.name(), villageId, providerServiceMapId); - List closed = diagnosticOrderRepo - .findBeneficiaryIdsClosed(type.name(), villageId, providerServiceMapId); - List awaitingManualEntry = diagnosticOrderRepo - .findBeneficiaryIdsAwaitingManualEntry(type.name(), villageId, providerServiceMapId); + List awaitingProviderResult = new ArrayList<>(); + List completed = new ArrayList<>(); + List failed = new ArrayList<>(); + List closed = new ArrayList<>(); + List awaitingManualEntry = new ArrayList<>(); + // One query instead of one per bucket; buckets are the same status groups the per-bucket + // queries used (matched case-insensitively, like the database comparison was). + for (Object[] row : diagnosticOrderRepo.findLatestOrderStatusByBeneficiary(type.name(), villageId, providerServiceMapId)) { + Long beneficiaryId = row[0] != null ? ((Number) row[0]).longValue() : null; + String status = row[1] != null ? row[1].toString().toUpperCase(java.util.Locale.ROOT) : ""; + switch (status) { + case "PENDING", "IN_PROGRESS" -> awaitingProviderResult.add(beneficiaryId); + case "COMPLETED" -> completed.add(beneficiaryId); + case "FAILED" -> failed.add(beneficiaryId); + case "CLOSED", "CANCELLED", "EXPIRED", "REFUSED" -> closed.add(beneficiaryId); + case "MANUAL_ENTRY" -> awaitingManualEntry.add(beneficiaryId); + default -> { } + } + } return new DiagnosticOrderStatusSummaryDto(awaitingProviderResult, completed, failed, closed, awaitingManualEntry); } diff --git a/src/main/java/com/iemr/flw/service/impl/TBSuspectedServiceImpl.java b/src/main/java/com/iemr/flw/service/impl/TBSuspectedServiceImpl.java index 1f4eca766..1f8fa0b8c 100644 --- a/src/main/java/com/iemr/flw/service/impl/TBSuspectedServiceImpl.java +++ b/src/main/java/com/iemr/flw/service/impl/TBSuspectedServiceImpl.java @@ -13,6 +13,8 @@ import com.iemr.flw.service.CampConfigService; import com.iemr.flw.service.TBStopVisitService; import com.iemr.flw.service.TBSuspectedService; +import com.iemr.flw.service.UserService; +import com.iemr.flw.masterEnum.StateCode; import org.modelmapper.ModelMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -21,7 +23,11 @@ import java.sql.Timestamp; import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Set; @Service public class TBSuspectedServiceImpl implements TBSuspectedService { @@ -40,6 +46,9 @@ public class TBSuspectedServiceImpl implements TBSuspectedService { @Autowired private IncentiveLogicService incentiveLogicService; + @Autowired + private UserService userService; + @Override public String getByBenId(Long benId, String authorisation) throws Exception { return null; @@ -140,22 +149,27 @@ public String save(TBSuspectedRequestDTO requestDTO) throws Exception { @Override public String getByUserId(GetBenRequestHandler request) { List dtos = new ArrayList<>(); - List tbSuspectedList = request.getProviderServiceMapID() != null + boolean stopTbVillage = request.getProviderServiceMapID() != null; + List tbSuspectedList = stopTbVillage ? tbSuspectedRepo.getByProviderServiceMapIdAndVillageId(request.getProviderServiceMapID(), request.getVillageID()) : tbSuspectedRepo.getByUserId(request.getAshaId(), request.getFromDate(), request.getToDate()); - for (TBSuspected tbSuspected : tbSuspectedList) { - TBSuspectedDTO dto = modelMapper.map(tbSuspected, TBSuspectedDTO.class); - dto.setUpdateDate(tbSuspected.getLastModDate()); - dto.setUpdatedBy(tbSuspected.getModifiedBy()); - dtos.add(dto); + if (stopTbVillage) { + dtos = stopTbVillageDtos(tbSuspectedList); + } else { + for (TBSuspected tbSuspected : tbSuspectedList) { + TBSuspectedDTO dto = modelMapper.map(tbSuspected, TBSuspectedDTO.class); + dto.setUpdateDate(tbSuspected.getLastModDate()); + dto.setUpdatedBy(tbSuspected.getModifiedBy()); + dtos.add(dto); - if (tbSuspected != null && Boolean.TRUE.equals(tbSuspected.getIsConfirmed())) { - incentiveLogicService.incentiveForTbSuspected( - tbSuspected.getBenId(), - tbSuspected.getVisitDate(), - tbSuspected.getVisitDate(), - tbSuspected.getUserId() - ); + if (tbSuspected != null && Boolean.TRUE.equals(tbSuspected.getIsConfirmed())) { + incentiveLogicService.incentiveForTbSuspected( + tbSuspected.getBenId(), + tbSuspected.getVisitDate(), + tbSuspected.getVisitDate(), + tbSuspected.getUserId() + ); + } } } TBSuspectedRequestDTO tbSuspectedRequestDTO = new TBSuspectedRequestDTO(); @@ -165,6 +179,74 @@ public String getByUserId(GetBenRequestHandler request) { return gson.toJson(tbSuspectedRequestDTO); } + // Stop TB village list (psm + village). Same DTOs and incentive calls as the per-user path, but the + // user's state is read once per user: IncentiveLogicImpl.incentiveForTbSuspected loads the full user + // detail on every call only to check for CG, and does nothing for any other state, so for other + // states the call is skipped and for CG users it is made exactly as before. + private List stopTbVillageDtos(List tbSuspectedList) { + List dtos = new ArrayList<>(); + Map stateByUser = new HashMap<>(); + Set usersWithoutDetail = new HashSet<>(); + for (TBSuspected tbSuspected : tbSuspectedList) { + TBSuspectedDTO dto = toDto(tbSuspected); + dto.setUpdateDate(tbSuspected.getLastModDate()); + dto.setUpdatedBy(tbSuspected.getModifiedBy()); + dtos.add(dto); + + if (Boolean.TRUE.equals(tbSuspected.getIsConfirmed())) { + Integer userId = tbSuspected.getUserId(); + if (!stateByUser.containsKey(userId) && !usersWithoutDetail.contains(userId)) { + try { + stateByUser.put(userId, userService.getUserDetail(userId).getStateId()); + } catch (Exception e) { + logger.error("Check Tb Suspected Incentive Exception: ", e); + usersWithoutDetail.add(userId); + } + } + Integer stateCode = stateByUser.get(userId); + if (stateCode != null && stateCode == StateCode.CG.getStateCode()) { + incentiveLogicService.incentiveForTbSuspected( + tbSuspected.getBenId(), + tbSuspected.getVisitDate(), + tbSuspected.getVisitDate(), + userId + ); + } + } + } + return dtos; + } + + // Field-by-field copy of the same-named fields ModelMapper copied (updateDate/updatedBy are set by + // the caller); ModelMapper's reflection per row was most of this list's time at village scale. + private static TBSuspectedDTO toDto(TBSuspected s) { + TBSuspectedDTO dto = new TBSuspectedDTO(); + dto.setId(s.getId()); + dto.setBenId(s.getBenId()); + dto.setVisitLabel(s.getVisitLabel()); + dto.setVisitDate(s.getVisitDate()); + dto.setTypeOfTBCase(s.getTypeOfTBCase()); + dto.setReasonForSuspicion(s.getReasonForSuspicion()); + dto.setIsSputumCollected(s.getIsSputumCollected()); + dto.setSputumSubmittedAt(s.getSputumSubmittedAt()); + dto.setNikshayId(s.getNikshayId()); + dto.setSputumTestResult(s.getSputumTestResult()); + dto.setIsChestXRayDone(s.getIsChestXRayDone()); + dto.setChestXRayResult(s.getChestXRayResult()); + dto.setReasonForRefusalXray(s.getReasonForRefusalXray()); + dto.setReasonForRefusalMTB(s.getReasonForRefusalMTB()); + dto.setReasonForRefusalMDRRIF(s.getReasonForRefusalMDRRIF()); + dto.setReasonForRefusalSputum(s.getReasonForRefusalSputum()); + dto.setMdrRifResult(s.getMdrRifResult()); + dto.setReferralFacility(s.getReferralFacility()); + dto.setIsTBConfirmed(s.getIsTBConfirmed()); + dto.setIsConfirmed(s.getIsConfirmed()); + dto.setIsDRTBConfirmed(s.getIsDRTBConfirmed()); + dto.setReferred(s.getReferred()); + dto.setFollowUps(s.getFollowUps()); + return dto; + } + @Override public java.util.Map getRecordsByIds(java.util.Collection ids) { Gson gson = new GsonBuilder().setDateFormat("MMM dd, yyyy h:mm:ss a").create(); diff --git a/src/main/java/com/iemr/flw/service/impl/WorklistPagePrefetch.java b/src/main/java/com/iemr/flw/service/impl/WorklistPagePrefetch.java new file mode 100644 index 000000000..b34acf9a8 --- /dev/null +++ b/src/main/java/com/iemr/flw/service/impl/WorklistPagePrefetch.java @@ -0,0 +1,191 @@ +package com.iemr.flw.service.impl; + +import com.iemr.flw.domain.iemr.BenAnthropometryDetail; +import com.iemr.flw.domain.iemr.BenFlowStatus; +import com.iemr.flw.domain.iemr.BenPhysicalVitalDetail; +import com.iemr.flw.domain.identity.*; +import jakarta.persistence.EntityManager; +import jakarta.persistence.PersistenceContext; +import org.springframework.stereotype.Component; + +import java.math.BigInteger; +import java.util.*; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * Loads everything the worklist builder needs for one page with one query per table, instead of + * ~16 queries per beneficiary. Each query keeps the ordering of the per-row query it replaces, so + * the response is unchanged. + * + * IDs are written into the SQL as numeric literals rather than bound as a collection: Hibernate 6.4.1 + * in this service throws "Binding is multi-valued" for every IN (:list) parameter. The values are + * Long/BigInteger read from the database, so the literal list cannot carry anything but digits. + */ +@Component +public class WorklistPagePrefetch { + + @PersistenceContext(unitName = "db_identity") + private EntityManager identityEm; + + @PersistenceContext(unitName = "db_iemr") + private EntityManager iemrEm; + + public static class Page { + final Map> mappingsByAddress = new HashMap<>(); + final Map details = new HashMap<>(); + final Map accounts = new HashMap<>(); + final Map images = new HashMap<>(); + final Map addresses = new HashMap<>(); + final Map contacts = new HashMap<>(); + final Map> benIdsByRegId = new HashMap<>(); + final Map> rmnchByRegId = new HashMap<>(); + final Map> bornBirthByRegId = new HashMap<>(); + final Map> householdsById = new HashMap<>(); + final Map> flowsByRegId = new HashMap<>(); + final Map> anthropometryByRegId = new HashMap<>(); + final Map> vitalsByRegId = new HashMap<>(); + final Set regIdsWithHealthId = new HashSet<>(); + final Map userIdByName = new HashMap<>(); + + public List mappings(BigInteger addressId) { + return mappingsByAddress.getOrDefault(addressId, Collections.emptyList()); + } + public RMNCHMBeneficiarydetail detail(BigInteger id) { return details.get(id); } + public RMNCHMBeneficiaryAccount account(BigInteger id) { return accounts.get(id); } + public RMNCHMBeneficiaryImage image(BigInteger id) { return images.get(id); } + public RMNCHMBeneficiaryaddress address(BigInteger id) { return addresses.get(id); } + public RMNCHMBeneficiarycontact contact(BigInteger id) { return contacts.get(id); } + public List benIds(Long regId) { return benIdsByRegId.getOrDefault(regId, Collections.emptyList()); } + public List rmnch(Long regId) { return rmnchByRegId.getOrDefault(regId, Collections.emptyList()); } + public List bornBirth(Long regId) { return bornBirthByRegId.getOrDefault(regId, Collections.emptyList()); } + public List households(Long householdId) { return householdsById.getOrDefault(householdId, Collections.emptyList()); } + public List flows(Long regId) { return flowsByRegId.getOrDefault(regId, Collections.emptyList()); } + public List anthropometry(Long regId) { return anthropometryByRegId.getOrDefault(regId, Collections.emptyList()); } + public List vitals(Long regId) { return vitalsByRegId.getOrDefault(regId, Collections.emptyList()); } + public boolean hasHealthId(Long regId) { return regIdsWithHealthId.contains(regId); } + public Map userIdCache() { return userIdByName; } + } + + /** The page's addresses in the order of {@code ids} (the worklist page order); ids with no row are skipped, as before. */ + public List addressesInOrder(List ids) { + Map byId = new HashMap<>(); + putAll(byId, entities(identityEm, RMNCHMBeneficiaryaddress.class, + "SELECT * FROM db_identity.i_beneficiaryaddress WHERE BenAddressID IN (%s)", + new LinkedHashSet<>(ids)), RMNCHMBeneficiaryaddress::getBenAddressID); + return ids.stream().map(byId::get).filter(Objects::nonNull).collect(Collectors.toList()); + } + + public Page load(List pageAddresses) { + Page page = new Page(); + pageAddresses.forEach(a -> page.addresses.put(a.getBenAddressID(), a)); + // The builder looks mappings up by address.getId(), which is the VanSerialNo column, not the + // address primary key - keep that exactly. + Set lookupIds = pageAddresses.stream().map(RMNCHMBeneficiaryaddress::getId) + .filter(Objects::nonNull).collect(Collectors.toCollection(LinkedHashSet::new)); + if (lookupIds.isEmpty()) return page; + + // getByAddressID: ORDER BY BenMapId DESC + for (RMNCHMBeneficiarymapping m : entities(identityEm, RMNCHMBeneficiarymapping.class, + "SELECT * FROM db_identity.i_beneficiarymapping WHERE BenAddressId IN (%s) ORDER BY BenMapId DESC", lookupIds)) { + page.mappingsByAddress.computeIfAbsent(m.getBenAddressId(), k -> new ArrayList<>()).add(m); + } + // the builder only uses the first (newest) mapping of each address + List used = page.mappingsByAddress.values().stream() + .map(l -> l.get(0)).collect(Collectors.toList()); + + putAll(page.details, entities(identityEm, RMNCHMBeneficiarydetail.class, + "SELECT * FROM db_identity.i_beneficiarydetails WHERE BeneficiaryDetailsId IN (%s)", + ids(used, RMNCHMBeneficiarymapping::getBenDetailsId)), RMNCHMBeneficiarydetail::getBeneficiaryDetailsId); + putAll(page.accounts, entities(identityEm, RMNCHMBeneficiaryAccount.class, + "SELECT * FROM db_identity.i_beneficiaryaccount WHERE BenAccountID IN (%s)", + ids(used, RMNCHMBeneficiarymapping::getBenAccountID)), RMNCHMBeneficiaryAccount::getBenAccountID); + putAll(page.images, entities(identityEm, RMNCHMBeneficiaryImage.class, + "SELECT * FROM db_identity.i_beneficiaryimage WHERE BenImageId IN (%s)", + ids(used, RMNCHMBeneficiarymapping::getBenImageId)), RMNCHMBeneficiaryImage::getBenImageId); + putAll(page.contacts, entities(identityEm, RMNCHMBeneficiarycontact.class, + "SELECT * FROM db_identity.i_beneficiarycontacts WHERE BenContactsID IN (%s)", + ids(used, RMNCHMBeneficiarymapping::getBenContactsId)), RMNCHMBeneficiarycontact::getBenContactsID); + Set missingAddressIds = ids(used, RMNCHMBeneficiarymapping::getBenAddressId); + missingAddressIds.removeAll(page.addresses.keySet()); + putAll(page.addresses, entities(identityEm, RMNCHMBeneficiaryaddress.class, + "SELECT * FROM db_identity.i_beneficiaryaddress WHERE BenAddressID IN (%s)", missingAddressIds), + RMNCHMBeneficiaryaddress::getBenAddressID); + + Set regIds = ids(used, RMNCHMBeneficiarymapping::getBenRegId); + if (regIds.isEmpty()) return page; + + for (Object[] r : rows(identityEm, + "SELECT BenRegId, BeneficiaryID FROM db_identity.m_beneficiaryregidmapping WHERE BenRegId IN (%s)", regIds)) { + page.benIdsByRegId.computeIfAbsent(toLong(r[0]), k -> new ArrayList<>()).add(toBigInteger(r[1])); + } + // getDetailsByRegID: ORDER BY beneficiaryDetails_RmnchId DESC + group(page.rmnchByRegId, entities(identityEm, RMNCHBeneficiaryDetailsRmnch.class, + "SELECT * FROM db_identity.i_beneficiarydetails_rmnch WHERE BeneficiaryRegID IN (%s) ORDER BY beneficiaryDetails_RmnchId DESC", + regIds), RMNCHBeneficiaryDetailsRmnch::getBenRegId); + group(page.bornBirthByRegId, entities(identityEm, RMNCHBornBirthDetails.class, + "SELECT * FROM db_identity.i_bornbirthdeatils WHERE BeneficiaryRegID IN (%s) ORDER BY BornBirthDeatilsId", + regIds), RMNCHBornBirthDetails::getBenRegId); + + // households of the newest rmnch row (the one the builder uses) + Set householdIds = page.rmnchByRegId.values().stream().map(l -> l.get(0).getHouseoldId()) + .filter(Objects::nonNull).collect(Collectors.toCollection(LinkedHashSet::new)); + group(page.householdsById, entities(identityEm, RMNCHHouseHoldDetails.class, + "SELECT * FROM db_identity.i_householddetails WHERE houseoldId IN (%s) ORDER BY houseHoldDetailsId", + householdIds), RMNCHHouseHoldDetails::getHouseoldId); + + group(page.flowsByRegId, entities(iemrEm, BenFlowStatus.class, + "SELECT * FROM db_iemr.i_ben_flow_outreach WHERE beneficiary_reg_id IN (%s) ORDER BY ben_flow_id", + regIds), BenFlowStatus::getBeneficiaryRegID); + // findByBeneficiaryRegIDOrderByCreatedDateDesc + group(page.anthropometryByRegId, entities(iemrEm, BenAnthropometryDetail.class, + "SELECT * FROM db_iemr.t_phy_anthropometry WHERE BeneficiaryRegID IN (%s) ORDER BY CreatedDate DESC", + regIds), BenAnthropometryDetail::getBeneficiaryRegID); + group(page.vitalsByRegId, entities(iemrEm, BenPhysicalVitalDetail.class, + "SELECT * FROM db_iemr.t_phy_vitals WHERE BeneficiaryRegID IN (%s) ORDER BY CreatedDate DESC", + regIds), BenPhysicalVitalDetail::getBeneficiaryRegID); + + for (Object id : iemrEm.createNativeQuery(String.format( + "SELECT DISTINCT BeneficiaryRegID FROM db_iemr.m_benhealthidmapping WHERE BeneficiaryRegID IN (%s)", + literalList(regIds))).getResultList()) { + page.regIdsWithHealthId.add(toLong(id)); + } + return page; + } + + @SuppressWarnings("unchecked") + private static List entities(EntityManager em, Class type, String sql, Collection ids) { + if (ids.isEmpty()) return Collections.emptyList(); + return em.createNativeQuery(String.format(sql, literalList(ids)), type).getResultList(); + } + + @SuppressWarnings("unchecked") + private static List rows(EntityManager em, String sql, Collection ids) { + if (ids.isEmpty()) return Collections.emptyList(); + return em.createNativeQuery(String.format(sql, literalList(ids))).getResultList(); + } + + private static Set ids(List rows, Function getter) { + return rows.stream().map(getter).filter(Objects::nonNull).collect(Collectors.toCollection(LinkedHashSet::new)); + } + + private static void putAll(Map target, List rows, Function key) { + rows.forEach(r -> target.putIfAbsent(key.apply(r), r)); + } + + private static void group(Map> target, List rows, Function key) { + rows.forEach(r -> target.computeIfAbsent(key.apply(r), k -> new ArrayList<>()).add(r)); + } + + // Only whole numbers ever reach the SQL text: anything else is rejected rather than quoted. + private static String literalList(Collection ids) { + return ids.stream().map(id -> { + if (id instanceof Number) return new BigInteger(id.toString()).toString(); + throw new IllegalArgumentException("Non-numeric id in worklist prefetch: " + id); + }).collect(Collectors.joining(",")); + } + + private static Long toLong(Object o) { return o == null ? null : ((Number) o).longValue(); } + + private static BigInteger toBigInteger(Object o) { return o == null ? null : new BigInteger(o.toString()); } +} diff --git a/src/main/java/com/iemr/flw/utils/ApiTimingFilter.java b/src/main/java/com/iemr/flw/utils/ApiTimingFilter.java new file mode 100644 index 000000000..d93159634 --- /dev/null +++ b/src/main/java/com/iemr/flw/utils/ApiTimingFilter.java @@ -0,0 +1,40 @@ +package com.iemr.flw.utils; + +import jakarta.servlet.Filter; +import jakarta.servlet.FilterChain; +import jakarta.servlet.ServletException; +import jakarta.servlet.ServletRequest; +import jakarta.servlet.ServletResponse; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.core.Ordered; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Component; + +import java.io.IOException; + +// One [API-TIMING] line per request (method, path, status, total ms incl. auth filters) so camp-laptop +// logs show every API's time: journalctl -u amrit-flw-api | grep API-TIMING +@Component +@Order(Ordered.HIGHEST_PRECEDENCE) +public class ApiTimingFilter implements Filter { + + private final Logger logger = LoggerFactory.getLogger(this.getClass().getName()); + + @Override + public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain chain) + throws IOException, ServletException { + + HttpServletRequest request = (HttpServletRequest) servletRequest; + HttpServletResponse response = (HttpServletResponse) servletResponse; + long start = System.currentTimeMillis(); + try { + chain.doFilter(servletRequest, servletResponse); + } finally { + logger.info("[API-TIMING] {} {} status={} ms={}", request.getMethod(), request.getRequestURI(), + response.getStatus(), System.currentTimeMillis() - start); + } + } +} diff --git a/src/main/java/com/iemr/flw/utils/response/OutputResponse.java b/src/main/java/com/iemr/flw/utils/response/OutputResponse.java index d4cb36705..7b5f7f8e3 100644 --- a/src/main/java/com/iemr/flw/utils/response/OutputResponse.java +++ b/src/main/java/com/iemr/flw/utils/response/OutputResponse.java @@ -83,6 +83,15 @@ public void setResponse(String message) { } + // Same result as setResponse(gson.toJson(x)) for an object/array, without writing the JSON text + // and parsing it back: pass gson.toJsonTree(x). + public void setResponse(JsonElement tree) { + this.data = tree; + statusCode = SUCCESS; + errorMessage = "Success"; + status = "Success"; + } + public void setError(Throwable thrown) { Date currDate = Calendar.getInstance().getTime(); logger.info("error happened due to " + thrown.getClass().getSimpleName() + " at " + currDate.toString()); diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 84cca4815..cf38a017c 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -35,7 +35,7 @@ spring.datasource.tomcat.remove-abandoned=true spring.datasource.tomcat.logAbandoned=true spring.datasource.continue-on-error=true spring.datasource.tomcat.remove-abandoned-timeout=600 -door-to-door-page-size=10 +door-to-door-page-size=50 #Get-HRP-Status get-HRP-Status=ANC/getHRPStatus From 358a5b028f62cc18a7050eee3ecc962389470fbb Mon Sep 17 00:00:00 2001 From: Mithun James <1007084+drtechie@users.noreply.github.com> Date: Thu, 29 Oct 2026 12:57:23 +0530 Subject: [PATCH 07/12] fix(stoptb): never move the feed past a change whose transaction is still open The safe point was the newest log line older than 5 s. seq is taken at insert, not at commit, so a transaction open longer than that could commit a lower seq after the feed had moved past it, and tablets would miss the change. The safe point is now below the oldest open transaction (information_schema.innodb_trx), minus the same 5 s margin; without the PROCESS privilege it falls back to the fixed lag and logs a warning once. Reproduced on MySQL 8.0: with A open holding seq 5 and B committed at seq 6, the old rule returned 6 (skipping 5); the new one returns 0 until A commits, then 6. Co-Authored-By: Claude Opus 5.5 --- .../service/impl/StopTBSyncServiceImpl.java | 32 ++++++++++++++++--- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/iemr/flw/service/impl/StopTBSyncServiceImpl.java b/src/main/java/com/iemr/flw/service/impl/StopTBSyncServiceImpl.java index 897e333a7..95613df85 100644 --- a/src/main/java/com/iemr/flw/service/impl/StopTBSyncServiceImpl.java +++ b/src/main/java/com/iemr/flw/service/impl/StopTBSyncServiceImpl.java @@ -45,7 +45,7 @@ public class StopTBSyncServiceImpl implements StopTBSyncService { * page carries at most this many; the token resumes from the last one sent. */ static final int MAX_BENEFICIARIES_PER_PAGE = 25; - /** Lines younger than this may still have earlier seqs uncommitted; they wait for the next call. */ + /** Margin under the oldest open transaction (see safePoint); lines younger than this wait. */ static final int SAFE_LAG_SECONDS = 5; /** The hub keeps 30 days of log; a token older than this may point at cleaned-up lines. */ static final long MAX_TOKEN_AGE_SECONDS = 29L * 24 * 3600; @@ -73,6 +73,7 @@ public class StopTBSyncServiceImpl implements StopTBSyncService { @Autowired private ObjectMapper objectMapper; + private volatile boolean innodbTrxWarned; private volatile Boolean supported; private volatile long supportCheckedAt; @@ -310,11 +311,32 @@ private String serverId() { return String.valueOf(em.createNativeQuery("SELECT @@server_uuid").getSingleResult()); } + /** + * The highest seq a reader may move past. seq is taken when a row is inserted, not when its + * transaction commits, so a transaction still open may hold a lower seq than lines already + * committed. Every line it writes is stamped at or after its start, so lines older than the oldest + * open transaction (information_schema.innodb_trx) are final; the lag also covers the gap between a + * statement's timestamp and its insert. A long open transaction holds the feed back until it ends. + */ private long safePoint() { - Object r = em.createNativeQuery("SELECT COALESCE(MAX(seq), 0) FROM db_iemr.t_sync_change_log " - + "WHERE changed_at <= NOW(6) - INTERVAL " + SAFE_LAG_SECONDS + " SECOND") - .getSingleResult(); - return ((Number) r).longValue(); + try { + Object r = em.createNativeQuery("SELECT COALESCE(MAX(seq), 0) FROM db_iemr.t_sync_change_log " + + "WHERE changed_at <= (SELECT LEAST(NOW(6), COALESCE(MIN(trx_started), NOW(6))) " + + "FROM information_schema.innodb_trx) - INTERVAL " + SAFE_LAG_SECONDS + " SECOND") + .getSingleResult(); + return ((Number) r).longValue(); + } catch (Exception e) { + // innodb_trx needs the PROCESS privilege; without it only the fixed lag protects late commits. + if (!innodbTrxWarned) { + logger.warn("STOP-TB changes feed: cannot read information_schema.innodb_trx ({}); " + + "using a fixed {} s lag only", e.getMessage(), SAFE_LAG_SECONDS); + innodbTrxWarned = true; + } + Object r = em.createNativeQuery("SELECT COALESCE(MAX(seq), 0) FROM db_iemr.t_sync_change_log " + + "WHERE changed_at <= NOW(6) - INTERVAL " + SAFE_LAG_SECONDS + " SECOND") + .getSingleResult(); + return ((Number) r).longValue(); + } } /** From 428c926236513fa9dd8c80c7ef5b0ed3d019f671 Mon Sep 17 00:00:00 2001 From: Mithun James <1007084+drtechie@users.noreply.github.com> Date: Thu, 29 Oct 2026 14:55:16 +0530 Subject: [PATCH 08/12] fix(diagnostics): polls that change nothing don't rewrite the order or result Every pending order was saved on every poll (every ~35 s): processed reset to 'N' so MMU re-uploaded it, and the change-log trigger re-sent it to every tablet. Now, when the vendor reports the same status, error, provider order id and result fields, only last_polled_at (and the retry count) is updated; a repeated poll error is recorded the same way. Real changes are saved as before. Pairs with AMRIT-DB V134 (PSMRI/AMRIT-DB#227). Co-Authored-By: Claude Opus 5.5 --- .../flw/repo/iemr/DiagnosticOrderRepo.java | 8 +++ .../impl/DiagnosticOrderServiceImpl.java | 56 +++++++++++++++++-- 2 files changed, 58 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/iemr/flw/repo/iemr/DiagnosticOrderRepo.java b/src/main/java/com/iemr/flw/repo/iemr/DiagnosticOrderRepo.java index 729e90f5a..36ecc6420 100644 --- a/src/main/java/com/iemr/flw/repo/iemr/DiagnosticOrderRepo.java +++ b/src/main/java/com/iemr/flw/repo/iemr/DiagnosticOrderRepo.java @@ -48,6 +48,14 @@ Optional findFirstByBeneficiaryIdAndOrderTypeAndDeletedFalseOrd @Query("UPDATE DiagnosticOrder o SET o.vanSerialNo = o.id WHERE o.id = :id") void updateVanSerialNo(@Param("id") Long id); + // A poll that changed nothing only records when it ran (and the retry count), leaving processed, + // modified_by and the rest of the row alone: no re-upload to central, no change-log line. + @Transactional + @Modifying + @Query("UPDATE DiagnosticOrder o SET o.lastPolledAt = :polledAt, o.retryCount = :retryCount WHERE o.id = :id") + int recordUnchangedPoll(@Param("id") Long id, @Param("polledAt") Timestamp polledAt, + @Param("retryCount") Integer retryCount); + // Beneficiary-status-summary queries, filtered the same way as // FormResponseRepo.findBeneficiaryIdsByFormIdAndStatusFiltered - null-safe optional // villageId/providerServiceMapId via a correlated BenFlowStatus subquery. diff --git a/src/main/java/com/iemr/flw/service/impl/DiagnosticOrderServiceImpl.java b/src/main/java/com/iemr/flw/service/impl/DiagnosticOrderServiceImpl.java index 3bc5c5a24..d169f20b3 100644 --- a/src/main/java/com/iemr/flw/service/impl/DiagnosticOrderServiceImpl.java +++ b/src/main/java/com/iemr/flw/service/impl/DiagnosticOrderServiceImpl.java @@ -454,7 +454,28 @@ public DiagnosticOrderResultDto processResult(DiagnosticOrder order, DiagnosticP private DiagnosticOrderResultDto processResult(DiagnosticOrder order, DiagnosticPollResult pollResult, boolean writeBackWhenClosed, String actingUser) throws Exception { Optional existingResult = diagnosticResultRepo.findByExternalOrderIdAndDeletedFalse(order.getExternalOrderId()); + // A poll that brings nothing new must not rewrite the rows: every rewrite resets processed + // (so MMU re-uploads the order) and would re-send it to every tablet. + boolean resultUnchanged = existingResult.isPresent() && sameResult(existingResult.get(), pollResult); + boolean orderUnchanged = sameOrderStatus(order, pollResult); DiagnosticResult result = existingResult.orElseGet(DiagnosticResult::new); + if (resultUnchanged && orderUnchanged && !writeBackWhenClosed) { + ingestAssets(order, pollResult); + diagnosticOrderRepo.recordUnchangedPoll(order.getId(), new Timestamp(System.currentTimeMillis()), + order.getRetryCount()); + DiagnosticOrderResultDto unchanged = new DiagnosticOrderResultDto(); + unchanged.setExternalOrderId(order.getExternalOrderId()); + unchanged.setOrderType(order.getOrderType()); + unchanged.setStatus(order.getStatus()); + unchanged.setProviderStatus(result.getProviderStatus()); + unchanged.setResultSummary(result.getResultSummary()); + unchanged.setErrorMessage(order.getErrorMessage()); + unchanged.setReasonToClose(order.getReasonToClose()); + unchanged.setTbPresence(result.getTbPresence()); + unchanged.setTbConfidence(result.getTbConfidence()); + unchanged.setDrugResistancePresence(result.getDrugResistancePresence()); + return unchanged; + } result.setExternalOrderId(order.getExternalOrderId()); result.setBeneficiaryId(order.getBeneficiaryId()); result.setProviderStatus(pollResult.getStatus().name()); @@ -579,12 +600,18 @@ public DiagnosticPollResult pollOnce(DiagnosticOrder order) throws Exception { try { return pollOnceLocked(fresh); } catch (Exception e) { - fresh.setRetryCount(fresh.getRetryCount() + 1); - fresh.setLastPolledAt(new Timestamp(System.currentTimeMillis())); - fresh.setErrorMessage(e.getMessage()); - fresh.setModifiedBy("SYSTEM"); - fresh.setProcessed("N"); - diagnosticOrderRepo.save(fresh); + int retries = (fresh.getRetryCount() == null ? 0 : fresh.getRetryCount()) + 1; + if (java.util.Objects.equals(fresh.getErrorMessage(), e.getMessage())) { + // Same failure as last time: only the attempt is recorded (see recordUnchangedPoll). + diagnosticOrderRepo.recordUnchangedPoll(fresh.getId(), new Timestamp(System.currentTimeMillis()), retries); + } else { + fresh.setRetryCount(retries); + fresh.setLastPolledAt(new Timestamp(System.currentTimeMillis())); + fresh.setErrorMessage(e.getMessage()); + fresh.setModifiedBy("SYSTEM"); + fresh.setProcessed("N"); + diagnosticOrderRepo.save(fresh); + } throw e; } }); @@ -616,6 +643,23 @@ public void expireOrder(DiagnosticOrder order, String reason) throws Exception { }); } + private static boolean sameResult(DiagnosticResult stored, DiagnosticPollResult polled) { + return java.util.Objects.equals(stored.getProviderStatus(), polled.getStatus() == null ? null : polled.getStatus().name()) + && java.util.Objects.equals(stored.getResultSummary(), polled.getResultSummary()) + && java.util.Objects.equals(stored.getTbPresence(), polled.getTbPresence()) + && java.util.Objects.equals(stored.getTbConfidence(), polled.getTbConfidence()) + && java.util.Objects.equals(stored.getDrugResistancePresence(), polled.getDrugResistancePresence()) + && (polled.getAssets() == null || polled.getAssets().isEmpty()); + } + + private static boolean sameOrderStatus(DiagnosticOrder order, DiagnosticPollResult polled) { + return polled.getStatus() != null + && polled.getStatus().name().equals(order.getStatus()) + && java.util.Objects.equals(order.getErrorMessage(), polled.getErrorMessage()) + && (polled.getProviderOrderId() == null + || polled.getProviderOrderId().equals(order.getProviderOrderId())); + } + private Optional findOutstandingOrder(Long orderId) { return diagnosticOrderRepo.findById(orderId) .filter(o -> !Boolean.TRUE.equals(o.getDeleted())) From 3eaaf717404b81eda9aaa8afe14d569f52e04b4d Mon Sep 17 00:00:00 2001 From: Mithun James <1007084+drtechie@users.noreply.github.com> Date: Thu, 29 Oct 2026 14:56:25 +0530 Subject: [PATCH 09/12] feat(stoptb): send a beneficiary's older records when they come into scope AMRIT-DB V134 logs flow-table scope changes (new flow row, village / service / deleted change) with op 'S'. When a delta page has such a line for a beneficiary, the feed now also sends that beneficiary's existing records of every requested type: they were logged before the beneficiary was in the tablet's villages, so they never passed the scope check and the token had already moved past them. Each type's query for this is derived from its snapshot query (village scope swapped for the given BenRegIds, paging dropped); StopTBSyncTypeTest checks every one. Co-Authored-By: Claude Opus 5.5 --- .../service/impl/StopTBSyncServiceImpl.java | 42 ++++++++++++++++++- .../iemr/flw/service/sync/StopTBSyncType.java | 21 ++++++++++ .../flw/service/sync/StopTBSyncTypeTest.java | 21 ++++++++++ 3 files changed, 82 insertions(+), 2 deletions(-) create mode 100644 src/test/java/com/iemr/flw/service/sync/StopTBSyncTypeTest.java diff --git a/src/main/java/com/iemr/flw/service/impl/StopTBSyncServiceImpl.java b/src/main/java/com/iemr/flw/service/impl/StopTBSyncServiceImpl.java index 95613df85..92639068e 100644 --- a/src/main/java/com/iemr/flw/service/impl/StopTBSyncServiceImpl.java +++ b/src/main/java/com/iemr/flw/service/impl/StopTBSyncServiceImpl.java @@ -170,8 +170,9 @@ private Map deltaPage(String serverId, long from, Scope scope, i // One row per record: its latest line in range, and that line's op. Deduplicates a record // changed many times since the token. List rows = em.createNativeQuery( - "SELECT l.entity, l.row_pk, l.seq, l.op FROM db_iemr.t_sync_change_log l JOIN (" - + " SELECT c.entity, c.row_pk, MAX(c.seq) AS last_seq FROM db_iemr.t_sync_change_log c " + "SELECT l.entity, l.row_pk, l.seq, l.op, g.scoped FROM db_iemr.t_sync_change_log l JOIN (" + + " SELECT c.entity, c.row_pk, MAX(c.seq) AS last_seq, MAX(c.op = 'S') AS scoped " + + " FROM db_iemr.t_sync_change_log c " + " WHERE c.seq > :from AND c.seq <= :safe AND c.entity IN (:entities) AND (" + " (c.ben_reg_id IS NOT NULL AND EXISTS (SELECT 1 FROM db_iemr.i_ben_flow_outreach f " + " WHERE f.beneficiary_reg_id = c.ben_reg_id AND f.providerServiceMapID = :psm AND f.villageID IN (:villages)))" @@ -210,6 +211,17 @@ private Map deltaPage(String serverId, long from, Scope scope, i keysByType.forEach((type, keys) -> built.put(type, build(type, keys, authorisation))); List> items = new ArrayList<>(); + // Beneficiaries that came into scope (op 'S': new flow row, or a village / service change): + // their records logged before that never passed the scope check, so send them now. + List scopedRegIds = new ArrayList<>(); + Map scopedSeq = new HashMap<>(); + for (Object[] r : rows) { + if (StopTBSyncType.BENEFICIARY.logEntity().equals(r[0]) && r[4] != null && ((Number) r[4]).intValue() == 1) { + long regId = ((Number) r[1]).longValue(); + scopedRegIds.add(regId); + scopedSeq.put(regId, ((Number) r[2]).longValue()); + } + } long lastSeq = from; for (Object[] r : rows) { lastSeq = ((Number) r[2]).longValue(); @@ -228,12 +240,38 @@ private Map deltaPage(String serverId, long from, Scope scope, i } } + if (!scopedRegIds.isEmpty()) { + long version = scopedSeq.values().stream().mapToLong(Long::longValue).max().orElse(lastSeq); + for (StopTBSyncType type : scope.types) { + if (type == StopTBSyncType.BENEFICIARY) continue; + List keys = keysForBeneficiaries(type, scope, scopedRegIds); + List alreadySent = keysByType.getOrDefault(type, List.of()); + keys.removeAll(alreadySent); + if (keys.isEmpty()) continue; + Map records = build(type, keys, authorisation); + for (Long key : keys) { + String data = records.get(key); + if (data != null) items.add(item(type, "upsert", key, version, data)); + } + } + } + boolean hasMore = full; long nextSeq = hasMore ? lastSeq : safe; return page(serverId, false, items, SyncToken.delta(serverId, nextSeq, System.currentTimeMillis() / 1000, scope.typesMask()), hasMore); } + @SuppressWarnings("unchecked") + private List keysForBeneficiaries(StopTBSyncType type, Scope scope, List regIds) { + jakarta.persistence.Query q = em.createNativeQuery(type.keysForBeneficiariesSql()).setParameter("regIds", regIds); + if (type.keysForBeneficiariesSql().contains(":psm")) q.setParameter("psm", scope.psm); + List rows = q.getResultList(); + List keys = new ArrayList<>(rows.size()); + for (Object r : rows) keys.add(((Number) r).longValue()); + return keys; + } + // ── Records, in the same JSON each type's existing endpoint returns ─────── private Map build(StopTBSyncType type, List keys, String authorisation) { diff --git a/src/main/java/com/iemr/flw/service/sync/StopTBSyncType.java b/src/main/java/com/iemr/flw/service/sync/StopTBSyncType.java index c0e91e953..8ac1db509 100644 --- a/src/main/java/com/iemr/flw/service/sync/StopTBSyncType.java +++ b/src/main/java/com/iemr/flw/service/sync/StopTBSyncType.java @@ -87,6 +87,11 @@ static final class Scope { static final String BEN_ID = "SELECT f.beneficiary_id FROM db_iemr.i_ben_flow_outreach f " + "WHERE f.providerServiceMapID = :psm AND f.villageID IN (:villages)"; static final String BEN_ID_NOT_DELETED = BEN_ID + " AND f.deleted = 0"; + // The same, for specific beneficiaries (by BenRegId) that just came into scope. + static final String REG_FOR_BENS = "SELECT f.beneficiary_reg_id FROM db_iemr.i_ben_flow_outreach f " + + "WHERE f.beneficiary_reg_id IN (:regIds)"; + static final String BEN_ID_FOR_BENS = "SELECT f.beneficiary_id FROM db_iemr.i_ben_flow_outreach f " + + "WHERE f.beneficiary_reg_id IN (:regIds)"; private Scope() { } @@ -95,12 +100,20 @@ private Scope() { private final String apiName; private final String logEntity; private final String snapshotKeySql; + private final String keysForBeneficiariesSql; private final List sourceTables; StopTBSyncType(String apiName, String logEntity, String snapshotKeySql, List sourceTables) { this.apiName = apiName; this.logEntity = logEntity; this.snapshotKeySql = snapshotKeySql; + // The snapshot query with its village scope swapped for "these beneficiaries" and its keyset + // paging dropped. Parameters: :regIds (and :psm where the snapshot filters on it). + this.keysForBeneficiariesSql = snapshotKeySql + .replace(Scope.BEN_ID_NOT_DELETED, Scope.BEN_ID_FOR_BENS) + .replace(Scope.BEN_ID, Scope.BEN_ID_FOR_BENS) + .replace(Scope.REG, Scope.REG_FOR_BENS) + .replaceAll(" AND \\S+ > :after ORDER BY \\S+ LIMIT :limit$", ""); this.sourceTables = sourceTables; } @@ -117,6 +130,14 @@ public String snapshotKeySql() { return snapshotKeySql; } + /** + * Keys of this type's records for beneficiaries that just came into a tablet's scope, so their + * older records (logged before they were in scope) still reach it. Not used for BENEFICIARY. + */ + public String keysForBeneficiariesSql() { + return keysForBeneficiariesSql; + } + public List sourceTables() { return sourceTables; } diff --git a/src/test/java/com/iemr/flw/service/sync/StopTBSyncTypeTest.java b/src/test/java/com/iemr/flw/service/sync/StopTBSyncTypeTest.java new file mode 100644 index 000000000..a7fa90e80 --- /dev/null +++ b/src/test/java/com/iemr/flw/service/sync/StopTBSyncTypeTest.java @@ -0,0 +1,21 @@ +package com.iemr.flw.service.sync; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +class StopTBSyncTypeTest { + + @Test + void childKeyQueriesTargetTheGivenBeneficiariesWithoutPaging() { + for (StopTBSyncType t : StopTBSyncType.values()) { + if (t == StopTBSyncType.BENEFICIARY) continue; + String sql = t.keysForBeneficiariesSql(); + assertTrue(sql.contains(":regIds"), t + ": " + sql); + assertFalse(sql.contains(":after"), t + ": " + sql); + assertFalse(sql.contains(":limit"), t + ": " + sql); + assertFalse(sql.contains(":villages"), t + ": " + sql); + System.out.println(t + " | " + sql); + } + } +} From 4b9c544ecb9f243e6be25830d5f9888b8ff086de Mon Sep 17 00:00:00 2001 From: Mithun James <1007084+drtechie@users.noreply.github.com> Date: Thu, 29 Oct 2026 14:58:36 +0530 Subject: [PATCH 10/12] feat(stoptb): changes feed only serves the user's own villages, all of them by default - The feed reads the user from the JwtToken header (required) and their services and villages from every active m_userservicerolemapping row. A service or village the user isn't mapped to is rejected; no villageIds means all of the user's villages for that service. Cached 5 minutes per user. - Tokens carry a hash of the service and village list, so a different scope starts a new snapshot instead of continuing from another scope's position. - getUserDetail no longer takes whichever mapping row came first: it uses the STOP-TB (Nikshay) row when there is one, merges villages across all rows for that service, and fails clearly when the user has no mapping instead of IndexOutOfBounds. UserSyncScopeTest covers merging, defaults and rejection; SyncTokenTest the new token. Co-Authored-By: Claude Opus 5.5 --- .../flw/controller/StopTBSyncController.java | 17 ++++- .../flw/repo/iemr/UserServiceRoleRepo.java | 5 ++ .../iemr/flw/service/StopTBSyncService.java | 4 +- .../service/impl/StopTBSyncServiceImpl.java | 45 ++++++++++--- .../flw/service/impl/UserServiceImpl.java | 40 ++++++++++- .../com/iemr/flw/service/sync/SyncToken.java | 38 +++++++---- .../iemr/flw/service/sync/UserSyncScope.java | 66 +++++++++++++++++++ .../iemr/flw/service/sync/SyncTokenTest.java | 12 ++-- .../flw/service/sync/UserSyncScopeTest.java | 42 ++++++++++++ 9 files changed, 235 insertions(+), 34 deletions(-) create mode 100644 src/main/java/com/iemr/flw/service/sync/UserSyncScope.java create mode 100644 src/test/java/com/iemr/flw/service/sync/UserSyncScopeTest.java diff --git a/src/main/java/com/iemr/flw/controller/StopTBSyncController.java b/src/main/java/com/iemr/flw/controller/StopTBSyncController.java index c367b857c..23201436f 100644 --- a/src/main/java/com/iemr/flw/controller/StopTBSyncController.java +++ b/src/main/java/com/iemr/flw/controller/StopTBSyncController.java @@ -3,6 +3,7 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.iemr.flw.service.StopTBSyncService; +import com.iemr.flw.utils.JwtUtil; import com.iemr.flw.utils.response.OutputResponse; import io.swagger.v3.oas.annotations.Operation; import org.slf4j.Logger; @@ -25,8 +26,13 @@ public class StopTBSyncController { @Autowired private StopTBSyncService stopTBSyncService; + @Autowired + private JwtUtil jwtUtil; + /** - * Body: { token: string|null, providerServiceMapID: int, villageIds: [int], types: [string], limit: int }. + * Headers: JwtToken (required; the user's role mappings decide the allowed service and villages). + * Body: { token: string|null, providerServiceMapID: int, villageIds: [int] (empty = all of the user's + * villages), types: [string], limit: int }. * types: the record types the app applies (all when omitted); changing it requires a new snapshot. * Response: { supported, serverId, snapshot, items: [{type, op, key, version, data}], nextToken, hasMore }. * Call again with nextToken while hasMore is true. On a server without the change log (AMRIT-DB before V132) the @@ -35,9 +41,14 @@ public class StopTBSyncController { @PostMapping("/changes") @Operation(summary = "STOP-TB records changed since the token (first call: everything in the villages)") public String getChanges(@RequestBody Map body, - @RequestHeader(value = "Authorization", required = false) String authorization) { + @RequestHeader(value = "Authorization", required = false) String authorization, + @RequestHeader(value = "JwtToken", required = false) String jwtToken) { OutputResponse response = new OutputResponse(); try { + // The user decides which villages may be synced, so the JWT is required here even + // where the request filter would let a mobile request through without one. + if (jwtToken == null || jwtToken.isBlank()) throw new IllegalArgumentException("JwtToken header is required"); + Integer userId = jwtUtil.extractUserId(jwtToken); Object token = body.get("token"); Integer psm = body.get("providerServiceMapID") != null ? Integer.parseInt(body.get("providerServiceMapID").toString()) : null; @@ -55,7 +66,7 @@ public String getChanges(@RequestBody Map body, } Integer limit = body.get("limit") != null ? (int) Double.parseDouble(body.get("limit").toString()) : null; Map data = stopTBSyncService.getChanges( - token != null ? token.toString() : null, psm, villageIds, types, limit, authorization); + userId, token != null ? token.toString() : null, psm, villageIds, types, limit, authorization); response.setResponse(gson.toJson(data)); } catch (Exception e) { logger.error("Error in sync/changes: " + e); diff --git a/src/main/java/com/iemr/flw/repo/iemr/UserServiceRoleRepo.java b/src/main/java/com/iemr/flw/repo/iemr/UserServiceRoleRepo.java index b94550a28..d2cdc2f46 100644 --- a/src/main/java/com/iemr/flw/repo/iemr/UserServiceRoleRepo.java +++ b/src/main/java/com/iemr/flw/repo/iemr/UserServiceRoleRepo.java @@ -60,6 +60,11 @@ public interface UserServiceRoleRepo extends JpaRepository getNikshayMappingRows(@Param("userId") Integer userId, @Param("providerServiceMapId") Integer providerServiceMapId); + // Every active mapping row's service and village list, for the STOP-TB changes feed's scope check. + @Query(value = "SELECT usrm.ProviderServiceMapID, usrm.Villageid FROM m_userservicerolemapping usrm " + + "WHERE usrm.UserID = :userId AND usrm.Deleted = false", nativeQuery = true) + List getServiceVillageRows(@Param("userId") Integer userId); + @Query(value = "SELECT NikshayTUID, TUName FROM m_nikshay_tu WHERE NikshayTUID IN (:ids)", nativeQuery = true) List findNikshayTuNames(@Param("ids") java.util.Collection ids); diff --git a/src/main/java/com/iemr/flw/service/StopTBSyncService.java b/src/main/java/com/iemr/flw/service/StopTBSyncService.java index a4023f118..9e8c94b6a 100644 --- a/src/main/java/com/iemr/flw/service/StopTBSyncService.java +++ b/src/main/java/com/iemr/flw/service/StopTBSyncService.java @@ -9,9 +9,11 @@ public interface StopTBSyncService { * One page of the STOP-TB changes feed for the given villages: a first full download when the * token is missing, from another hub or too old; otherwise only what changed after it. * Returns {supported:false} where the change log isn't installed, so the app keeps its full pull. + * The service and villages must be mapped to userId (from the JWT); empty villageIds means all of + * the user's villages for that service. * types limits the feed to the record types the app applies (StopTBSyncType api names); null or * empty means all. The app must start a new snapshot (no token) whenever it changes this list. */ - Map getChanges(String token, Integer providerServiceMapID, List villageIds, + Map getChanges(Integer userId, String token, Integer providerServiceMapID, List villageIds, List types, Integer limit, String authorisation); } diff --git a/src/main/java/com/iemr/flw/service/impl/StopTBSyncServiceImpl.java b/src/main/java/com/iemr/flw/service/impl/StopTBSyncServiceImpl.java index 92639068e..4b144d60a 100644 --- a/src/main/java/com/iemr/flw/service/impl/StopTBSyncServiceImpl.java +++ b/src/main/java/com/iemr/flw/service/impl/StopTBSyncServiceImpl.java @@ -13,6 +13,8 @@ import com.iemr.flw.service.TBSuspectedService; import com.iemr.flw.service.sync.StopTBSyncType; import com.iemr.flw.service.sync.SyncToken; +import com.iemr.flw.service.sync.UserSyncScope; +import com.iemr.flw.repo.iemr.UserServiceRoleRepo; import jakarta.persistence.EntityManager; import jakarta.persistence.PersistenceContext; import org.slf4j.Logger; @@ -73,20 +75,29 @@ public class StopTBSyncServiceImpl implements StopTBSyncService { @Autowired private ObjectMapper objectMapper; + private static final long USER_SCOPE_TTL_MS = 5 * 60 * 1000L; + private final Map> userScopes = new java.util.concurrent.ConcurrentHashMap<>(); + + @Autowired + private UserServiceRoleRepo userServiceRoleRepo; + private volatile boolean innodbTrxWarned; private volatile Boolean supported; private volatile long supportCheckedAt; @Override - public Map getChanges(String token, Integer providerServiceMapID, List villageIds, - List types, Integer limit, String authorisation) { + public Map getChanges(Integer userId, String token, Integer providerServiceMapID, + List villageIds, List types, Integer limit, + String authorisation) { if (!isSupported()) { Map out = new LinkedHashMap<>(); out.put("supported", false); return out; } + if (userId == null) throw new IllegalArgumentException("A valid JwtToken is required"); if (providerServiceMapID == null) throw new IllegalArgumentException("providerServiceMapID is required"); - if (villageIds == null || villageIds.isEmpty()) throw new IllegalArgumentException("villageIds is required"); + // Only the user's own service and villages; none asked for means all of them. + List villages = userScope(userId).resolve(providerServiceMapID, villageIds); int pageSize = limit == null || limit <= 0 ? DEFAULT_LIMIT : Math.min(limit, MAX_LIMIT); String serverId = serverId(); @@ -95,12 +106,12 @@ public Map getChanges(String token, Integer providerServiceMapID boolean usable = parsed != null && serverId.equals(parsed.serverId()) && (parsed.kind() == SyncToken.Kind.SNAPSHOT || now - parsed.issuedAt() <= MAX_TOKEN_AGE_SECONDS); - Scope scope = new Scope(providerServiceMapID, villageIds, requestedTypes(types)); - // A token covers only the types it was issued for; newly requested types need a snapshot. - usable = usable && parsed.typesMask() == scope.typesMask(); + Scope scope = new Scope(providerServiceMapID, villages, requestedTypes(types)); + // A token covers only the types and villages it was issued for; anything else needs a snapshot. + usable = usable && parsed.typesMask() == scope.typesMask() && parsed.scopeHash() == scope.scopeHash(); if (!usable) { if (parsed != null) logger.info("Sync token {} not usable here, starting a snapshot", parsed); - return snapshotPage(serverId, SyncToken.snapshot(serverId, safePoint(), 0, 0, scope.typesMask()), + return snapshotPage(serverId, SyncToken.snapshot(serverId, safePoint(), 0, 0, scope.typesMask(), scope.scopeHash()), scope, pageSize, authorisation); } return parsed.kind() == SyncToken.Kind.SNAPSHOT @@ -143,8 +154,8 @@ private Map snapshotPage(String serverId, SyncToken at, Scope sc boolean done = typeIndex >= types.length; SyncToken next = done - ? SyncToken.delta(serverId, at.seq(), System.currentTimeMillis() / 1000, scope.typesMask()) - : SyncToken.snapshot(serverId, at.seq(), typeIndex, after, scope.typesMask()); + ? SyncToken.delta(serverId, at.seq(), System.currentTimeMillis() / 1000, scope.typesMask(), scope.scopeHash()) + : SyncToken.snapshot(serverId, at.seq(), typeIndex, after, scope.typesMask(), scope.scopeHash()); return page(serverId, true, items, next, !done); } @@ -259,7 +270,7 @@ private Map deltaPage(String serverId, long from, Scope scope, i boolean hasMore = full; long nextSeq = hasMore ? lastSeq : safe; return page(serverId, false, items, - SyncToken.delta(serverId, nextSeq, System.currentTimeMillis() / 1000, scope.typesMask()), hasMore); + SyncToken.delta(serverId, nextSeq, System.currentTimeMillis() / 1000, scope.typesMask(), scope.scopeHash()), hasMore); } @SuppressWarnings("unchecked") @@ -343,6 +354,16 @@ private Map page(String serverId, boolean snapshot, List cached = userScopes.get(userId); + if (cached != null && now - cached.getKey() < USER_SCOPE_TTL_MS) return cached.getValue(); + UserSyncScope scope = UserSyncScope.fromMappingRows(userServiceRoleRepo.getServiceVillageRows(userId)); + userScopes.put(userId, Map.entry(now, scope)); + return scope; + } + // ── Hub state ───────────────────────────────────────────────────────────── private String serverId() { @@ -443,6 +464,10 @@ private static final class Scope { this.types = types; } + long scopeHash() { + return UserSyncScope.scopeHash(psm, villages); + } + long typesMask() { long mask = 0; for (StopTBSyncType t : types) mask |= 1L << t.ordinal(); diff --git a/src/main/java/com/iemr/flw/service/impl/UserServiceImpl.java b/src/main/java/com/iemr/flw/service/impl/UserServiceImpl.java index 7537e5917..b3fef7139 100644 --- a/src/main/java/com/iemr/flw/service/impl/UserServiceImpl.java +++ b/src/main/java/com/iemr/flw/service/impl/UserServiceImpl.java @@ -28,7 +28,15 @@ public class UserServiceImpl implements UserService { public UserServiceRoleDTO getUserDetail(Integer userId) { logger.info("calling getUserRole for userId: " + userId); - UserServiceRoleDTO userRole = userServiceRoleRepo.getUserRole(userId).get(0); + List roles = userServiceRoleRepo.getUserRole(userId); + if (roles == null || roles.isEmpty()) { + throw new IllegalStateException("No active role mapping for userId " + userId); + } + // A user can have several mapping rows. Taking whichever came first dropped the villages + // on the others; use the STOP-TB (Nikshay) row when there is one, and the villages of + // every row for that same service. + UserServiceRoleDTO userRole = primaryRole(userId, roles); + mergeVillages(userRole, roles); userRole.setFacilityData(facilityDataService.buildFacilityData(userId, userRole.getRoleName())); // Two different villages can legitimately share the same name (confirmed live: @@ -103,6 +111,36 @@ public UserServiceRoleDTO getUserDetail(Integer userId) { return userRole; } + private UserServiceRoleDTO primaryRole(Integer userId, List roles) { + if (roles.size() == 1) return roles.get(0); + for (UserServiceRoleDTO role : roles) { + List nikshay = userServiceRoleRepo.getNikshayMappingRows(userId, role.getProviderServiceMapId()); + if (nikshay != null && !nikshay.isEmpty()) return role; + } + return roles.get(0); + } + + // Unions the parallel villageId/villageName lists of every row for the chosen row's service, + // keeping first-seen order and one entry per village ID. Rows whose two lists don't line up + // are left out of the merge rather than risk pairing an ID with the wrong name. + private void mergeVillages(UserServiceRoleDTO userRole, List roles) { + java.util.LinkedHashMap byId = new java.util.LinkedHashMap<>(); + for (UserServiceRoleDTO role : roles) { + if (!java.util.Objects.equals(role.getProviderServiceMapId(), userRole.getProviderServiceMapId())) continue; + if (role.getVillageId() == null || role.getVillageId().isBlank() + || role.getVillageName() == null || role.getVillageName().isBlank()) continue; + String[] ids = role.getVillageId().split(",", -1); + String[] names = role.getVillageName().split(",", -1); + if (ids.length != names.length) continue; + for (int i = 0; i < ids.length; i++) { + if (!ids[i].isBlank()) byId.putIfAbsent(ids[i].trim(), names[i].trim()); + } + } + if (byId.isEmpty()) return; + userRole.setVillageId(String.join(",", byId.keySet())); + userRole.setVillageName(String.join(",", byId.values())); + } + // Zips the parallel villageId/villageName comma lists and rejoins each pair as // "name (id)", e.g. "Sanapindapadar,Sanapindapadar" + "275135,275136" becomes // "Sanapindapadar (275135),Sanapindapadar (275136)". Falls back to the original, diff --git a/src/main/java/com/iemr/flw/service/sync/SyncToken.java b/src/main/java/com/iemr/flw/service/sync/SyncToken.java index e64ac3888..0055b0247 100644 --- a/src/main/java/com/iemr/flw/service/sync/SyncToken.java +++ b/src/main/java/com/iemr/flw/service/sync/SyncToken.java @@ -3,16 +3,18 @@ /** * The tablet's opaque sync position. Two kinds: * - * d.<serverId>.<seq>.<issuedEpochSeconds>.<typesMask> + * d.<serverId>.<seq>.<issuedEpochSeconds>.<typesMask>.<scopeHash> * changes after change-log line seq on this hub; - * s.<serverId>.<snapshotSeq>.<typeIndex>.<afterKey>.<typesMask> + * s.<serverId>.<snapshotSeq>.<typeIndex>.<afterKey>.<typesMask>.<scopeHash> * part-way through a first download: the next page starts at type typeIndex, after afterKey, * and when it finishes the tablet continues with changes after snapshotSeq. * * serverId is the hub's MySQL server_uuid, so a token from another hub is never trusted. The issue * time lets the hub tell when the log lines a token points after may have been cleaned up. * typesMask is the set of record types the token was issued for (bit = StopTBSyncType ordinal): a - * token only covers those types, so a request for a different set starts a new snapshot. + * token only covers those types, so a request for a different set starts a new snapshot. scopeHash + * is the service and village list the token was issued for (UserSyncScope.scopeHash); a different + * scope also starts a new snapshot. * * @author Piramal Swasthya */ @@ -27,9 +29,10 @@ public enum Kind { DELTA, SNAPSHOT } private final int typeIndex; private final long afterKey; private final long typesMask; + private final long scopeHash; private SyncToken(Kind kind, String serverId, long seq, long issuedAt, int typeIndex, long afterKey, - long typesMask) { + long typesMask, long scopeHash) { this.kind = kind; this.serverId = serverId; this.seq = seq; @@ -37,14 +40,16 @@ private SyncToken(Kind kind, String serverId, long seq, long issuedAt, int typeI this.typeIndex = typeIndex; this.afterKey = afterKey; this.typesMask = typesMask; + this.scopeHash = scopeHash; } - public static SyncToken delta(String serverId, long seq, long issuedAt, long typesMask) { - return new SyncToken(Kind.DELTA, serverId, seq, issuedAt, 0, 0, typesMask); + public static SyncToken delta(String serverId, long seq, long issuedAt, long typesMask, long scopeHash) { + return new SyncToken(Kind.DELTA, serverId, seq, issuedAt, 0, 0, typesMask, scopeHash); } - public static SyncToken snapshot(String serverId, long snapshotSeq, int typeIndex, long afterKey, long typesMask) { - return new SyncToken(Kind.SNAPSHOT, serverId, snapshotSeq, 0, typeIndex, afterKey, typesMask); + public static SyncToken snapshot(String serverId, long snapshotSeq, int typeIndex, long afterKey, long typesMask, + long scopeHash) { + return new SyncToken(Kind.SNAPSHOT, serverId, snapshotSeq, 0, typeIndex, afterKey, typesMask, scopeHash); } /** null for a missing or unreadable token; the caller then starts a snapshot. */ @@ -52,12 +57,13 @@ public static SyncToken parse(String token) { if (token == null || token.isBlank()) return null; String[] p = token.trim().split("\\."); try { - if (p.length == 5 && "d".equals(p[0]) && !p[1].isEmpty()) { - return delta(p[1], Long.parseLong(p[2]), Long.parseLong(p[3]), Long.parseLong(p[4])); + if (p.length == 6 && "d".equals(p[0]) && !p[1].isEmpty()) { + return delta(p[1], Long.parseLong(p[2]), Long.parseLong(p[3]), Long.parseLong(p[4]), + Long.parseLong(p[5])); } - if (p.length == 6 && "s".equals(p[0]) && !p[1].isEmpty()) { + if (p.length == 7 && "s".equals(p[0]) && !p[1].isEmpty()) { return snapshot(p[1], Long.parseLong(p[2]), Integer.parseInt(p[3]), Long.parseLong(p[4]), - Long.parseLong(p[5])); + Long.parseLong(p[5]), Long.parseLong(p[6])); } } catch (NumberFormatException e) { return null; @@ -67,8 +73,8 @@ public static SyncToken parse(String token) { public String encode() { return kind == Kind.DELTA - ? "d." + serverId + "." + seq + "." + issuedAt + "." + typesMask - : "s." + serverId + "." + seq + "." + typeIndex + "." + afterKey + "." + typesMask; + ? "d." + serverId + "." + seq + "." + issuedAt + "." + typesMask + "." + scopeHash + : "s." + serverId + "." + seq + "." + typeIndex + "." + afterKey + "." + typesMask + "." + scopeHash; } public Kind kind() { @@ -100,6 +106,10 @@ public long typesMask() { return typesMask; } + public long scopeHash() { + return scopeHash; + } + @Override public String toString() { return encode(); diff --git a/src/main/java/com/iemr/flw/service/sync/UserSyncScope.java b/src/main/java/com/iemr/flw/service/sync/UserSyncScope.java new file mode 100644 index 000000000..0573daf04 --- /dev/null +++ b/src/main/java/com/iemr/flw/service/sync/UserSyncScope.java @@ -0,0 +1,66 @@ +package com.iemr.flw.service.sync; + +import java.util.*; + +/** + * The villages a user may sync, per provider service map: every active role-mapping row counts + * (a user can have several), each holding a comma-separated village list. + * + * @author Piramal Swasthya + */ +public final class UserSyncScope { + + private final Map> villagesByService; + + private UserSyncScope(Map> villagesByService) { + this.villagesByService = villagesByService; + } + + /** rows: [ProviderServiceMapID, Villageid csv] from m_userservicerolemapping. */ + public static UserSyncScope fromMappingRows(List rows) { + Map> out = new HashMap<>(); + for (Object[] row : rows) { + if (row == null || row[0] == null) continue; + Set villages = out.computeIfAbsent(((Number) row[0]).intValue(), k -> new TreeSet<>()); + if (row.length > 1 && row[1] != null) { + for (String id : row[1].toString().split(",")) { + try { + villages.add(Integer.valueOf(id.trim())); + } catch (NumberFormatException ignored) { + // malformed entry: not a village the user can sync + } + } + } + } + return new UserSyncScope(out); + } + + /** + * The villages to sync for this request: all of the user's villages for the service when none + * are asked for, otherwise exactly the ones asked for. Throws when the service or any village + * isn't mapped to the user. + */ + public List resolve(Integer providerServiceMapID, List requested) { + Set allowed = villagesByService.get(providerServiceMapID); + if (allowed == null) { + throw new IllegalArgumentException("User is not mapped to providerServiceMapID " + providerServiceMapID); + } + if (requested == null || requested.isEmpty()) { + if (allowed.isEmpty()) throw new IllegalArgumentException("User has no villages for this service"); + return new ArrayList<>(allowed); + } + List outside = new ArrayList<>(); + for (Integer v : requested) if (!allowed.contains(v)) outside.add(v); + if (!outside.isEmpty()) { + throw new IllegalArgumentException("Villages not assigned to the user: " + outside); + } + return new ArrayList<>(new TreeSet<>(requested)); + } + + /** A stable fingerprint of a service and its village list, bound into sync tokens. */ + public static long scopeHash(Integer providerServiceMapID, List villages) { + List sorted = new ArrayList<>(villages); + Collections.sort(sorted); + return Integer.toUnsignedLong((providerServiceMapID + "|" + sorted).hashCode()); + } +} diff --git a/src/test/java/com/iemr/flw/service/sync/SyncTokenTest.java b/src/test/java/com/iemr/flw/service/sync/SyncTokenTest.java index bb72b31c7..3f247605b 100644 --- a/src/test/java/com/iemr/flw/service/sync/SyncTokenTest.java +++ b/src/test/java/com/iemr/flw/service/sync/SyncTokenTest.java @@ -10,22 +10,24 @@ class SyncTokenTest { @Test void deltaRoundTrips() { - SyncToken t = SyncToken.parse(SyncToken.delta(UUID, 1000, 1761700000L, 0b101).encode()); + SyncToken t = SyncToken.parse(SyncToken.delta(UUID, 1000, 1761700000L, 0b101, 42L).encode()); assertEquals(SyncToken.Kind.DELTA, t.kind()); assertEquals(UUID, t.serverId()); assertEquals(1000, t.seq()); assertEquals(1761700000L, t.issuedAt()); assertEquals(0b101, t.typesMask()); + assertEquals(42L, t.scopeHash()); } @Test void snapshotRoundTrips() { - SyncToken t = SyncToken.parse(SyncToken.snapshot(UUID, 5230, 3, 900, 7).encode()); + SyncToken t = SyncToken.parse(SyncToken.snapshot(UUID, 5230, 3, 900, 7, 99L).encode()); assertEquals(SyncToken.Kind.SNAPSHOT, t.kind()); assertEquals(5230, t.seq()); assertEquals(3, t.typeIndex()); assertEquals(900, t.afterKey()); assertEquals(7, t.typesMask()); + assertEquals(99L, t.scopeHash()); } @Test @@ -33,9 +35,9 @@ void unreadableTokensMeanStartOver() { assertNull(SyncToken.parse(null)); assertNull(SyncToken.parse("")); assertNull(SyncToken.parse("garbage")); - assertNull(SyncToken.parse("d." + UUID + ".x.1.1")); - assertNull(SyncToken.parse("d." + UUID + ".1.1")); // no types: issued before types were bound - assertNull(SyncToken.parse("d..1.1.1")); + assertNull(SyncToken.parse("d." + UUID + ".x.1.1.1")); + assertNull(SyncToken.parse("d." + UUID + ".1.1.1")); // no scope: issued before scope was bound + assertNull(SyncToken.parse("d..1.1.1.1")); assertNull(SyncToken.parse("q." + UUID + ".1.1")); } diff --git a/src/test/java/com/iemr/flw/service/sync/UserSyncScopeTest.java b/src/test/java/com/iemr/flw/service/sync/UserSyncScopeTest.java new file mode 100644 index 000000000..1398bbbdf --- /dev/null +++ b/src/test/java/com/iemr/flw/service/sync/UserSyncScopeTest.java @@ -0,0 +1,42 @@ +package com.iemr.flw.service.sync; + +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +class UserSyncScopeTest { + + // Two mapping rows for service 1738 (villages merge), one for 1744, and a malformed entry. + private final UserSyncScope scope = UserSyncScope.fromMappingRows(List.of( + new Object[]{1738, "5443, 5444"}, + new Object[]{1738, "5445,x"}, + new Object[]{1744, "6677"}, + new Object[]{1750, null})); + + @Test + void noVillagesAskedForMeansAllOfTheUsersVillagesForThatService() { + assertEquals(List.of(5443, 5444, 5445), scope.resolve(1738, List.of())); + assertEquals(List.of(6677), scope.resolve(1744, null)); + } + + @Test + void askedForVillagesMustAllBeAssigned() { + assertEquals(List.of(5443, 5445), scope.resolve(1738, List.of(5445, 5443, 5445))); + assertThrows(IllegalArgumentException.class, () -> scope.resolve(1738, List.of(5443, 6677))); + } + + @Test + void unmappedServiceOrServiceWithoutVillagesIsRejected() { + assertThrows(IllegalArgumentException.class, () -> scope.resolve(9999, List.of())); + assertThrows(IllegalArgumentException.class, () -> scope.resolve(1750, List.of())); + } + + @Test + void scopeHashIgnoresVillageOrderButNotContent() { + assertEquals(UserSyncScope.scopeHash(1738, List.of(5443, 5444)), UserSyncScope.scopeHash(1738, List.of(5444, 5443))); + assertNotEquals(UserSyncScope.scopeHash(1738, List.of(5443)), UserSyncScope.scopeHash(1738, List.of(5443, 5444))); + assertNotEquals(UserSyncScope.scopeHash(1738, List.of(5443)), UserSyncScope.scopeHash(1744, List.of(5443))); + } +} From 2f07c7363b5a8b6033d5e693a81c0f0a6c93210a Mon Sep 17 00:00:00 2001 From: Mithun James <1007084+drtechie@users.noreply.github.com> Date: Thu, 29 Oct 2026 15:30:15 +0530 Subject: [PATCH 11/12] fix(diagnostic): don't treat a failed result update as a lost race processResult caught every DataIntegrityViolationException on the result save as a lost insert race and went on to mark the order COMPLETED. When the save of an existing result failed (V134 trigger NULL row_pk), the order completed with its result still PENDING. Only a first insert can race; rethrow otherwise so the order is left as it was. Co-Authored-By: Claude Opus 5.5 --- .../com/iemr/flw/service/impl/DiagnosticOrderServiceImpl.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/com/iemr/flw/service/impl/DiagnosticOrderServiceImpl.java b/src/main/java/com/iemr/flw/service/impl/DiagnosticOrderServiceImpl.java index d169f20b3..913a747a2 100644 --- a/src/main/java/com/iemr/flw/service/impl/DiagnosticOrderServiceImpl.java +++ b/src/main/java/com/iemr/flw/service/impl/DiagnosticOrderServiceImpl.java @@ -501,6 +501,9 @@ private DiagnosticOrderResultDto processResult(DiagnosticOrder order, Diagnostic diagnosticResultRepo.save(result); if (result.getVanSerialNo() == null) diagnosticResultRepo.updateVanSerialNo(result.getId()); } catch (DataIntegrityViolationException dive) { + // Only a first insert can lose a race (unique external_order_id). Any other integrity + // failure means the result was not saved, so the order must not move on without it. + if (existingResult.isPresent()) throw dive; logger.warn("Lost result upsert race for externalOrderId={}", order.getExternalOrderId()); result = diagnosticResultRepo.findByExternalOrderIdAndDeletedFalse(order.getExternalOrderId()).orElse(result); } From 926547c4d6b6c23b6aa6025b4f3ea5a3f704713d Mon Sep 17 00:00:00 2001 From: Vishwanath Balkur <118195001+vishwab1@users.noreply.github.com> Date: Thu, 29 Oct 2026 15:59:38 +0530 Subject: [PATCH 12/12] feat(stoptb): save OPD prescription without dispensing by default (#395) Inventory is not part of this release, so the Stop TB OPD save no longer checks the camp store or stock, writes no patient issue and leaves stock unchanged. Drugs from the app's own list are saved as sent (DrugID, name, form, frequency, duration, instructions); quantity is worked out when the app sends 0. A new save for the visit replaces its earlier prescription, so an edit or retry leaves one copy. The existing dispensing path is kept behind stoptb.dispense.enabled (default false, no config needed) for the release that brings inventory. Co-authored-by: Claude Opus 5.5 --- .../service/StopTBPrescriptionService.java | 77 ++++++++++++++++++- 1 file changed, 74 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/iemr/flw/service/StopTBPrescriptionService.java b/src/main/java/com/iemr/flw/service/StopTBPrescriptionService.java index 740057107..c61b4d4f9 100644 --- a/src/main/java/com/iemr/flw/service/StopTBPrescriptionService.java +++ b/src/main/java/com/iemr/flw/service/StopTBPrescriptionService.java @@ -10,6 +10,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; @@ -48,6 +49,11 @@ public class StopTBPrescriptionService { @Autowired private BeneficiaryRepo beneficiaryRepo; + // Off by default: save the prescription only (no store/stock check, no patient issue, stock unchanged). + // true = check stock and dispense from the camp store. + @Value("${stoptb.dispense.enabled:false}") + private boolean dispenseEnabled; + private final Logger logger = LoggerFactory.getLogger(StopTBPrescriptionService.class); public static class InsufficientStockException extends RuntimeException { @@ -150,6 +156,10 @@ public void saveChiefComplaints(List> complaints, Long benef public Map prescribeAndDispense(Map prescription, Long beneficiaryRegID, BenVisitDetail visit, String submissionId, String createdBy, Integer vanID, Integer parkingPlaceID) { List> drugs = toList(prescription.get("drugs")); + if (!dispenseEnabled) { + return prescribeWithoutDispensing(prescription, drugs, beneficiaryRegID, visit, createdBy, vanID, + parkingPlaceID); + } Map result = new LinkedHashMap<>(); if (drugs.isEmpty()) { // Remarks without medicine are still kept, as MMU saves the visit instruction without drugs. @@ -176,7 +186,7 @@ public Map prescribeAndDispense(Map prescription Integer qty = toInt(d.get("qtyPrescribed")); if (qty == null || qty <= 0) { // No quantity from the app: tablets/capsules = doses per day x days, other forms = 1. - qty = calculateQuantity(d, items.get(drugID)); + qty = calculateQuantity(d, toInt(items.get(drugID)[7])); if (qty == null) { throw new IllegalArgumentException("qtyPrescribed is required for drugID " + drugID + " (tablet/capsule quantity needs frequency and duration)"); @@ -480,6 +490,68 @@ private Object[] getPatientDetails(Long beneficiaryRegID) { } } + /** + * Dispensing off (stoptb.dispense.enabled=false): the prescription is saved as the app sent it. No store or + * stock check and no patient issue, so drugs from the app's own list need not exist in this camp's store; + * DrugID, name and form are taken from the request. A new save replaces the visit's earlier prescription + * (same as chief complaints), so an edit or retry does not leave a second copy. + */ + private Map prescribeWithoutDispensing(Map prescription, + List> drugs, Long beneficiaryRegID, BenVisitDetail visit, String createdBy, + Integer vanID, Integer parkingPlaceID) { + for (Map d : drugs) { + if (toInt(d.get("drugID")) == null) throw new IllegalArgumentException("drugID is required for every drug"); + } + // Processed='N' so the removal is uploaded to central again (central updates rows it already has). + for (String table : new String[] { "t_prescribeddrug", "t_prescription" }) { + em.createNativeQuery("UPDATE db_iemr." + table + " SET Deleted = true, Processed = 'N', ModifiedBy = :user " + + "WHERE BeneficiaryRegID = :ben AND VisitCode = :visitCode AND Deleted = false") + .setParameter("user", createdBy) + .setParameter("ben", beneficiaryRegID) + .setParameter("visitCode", visit.getVisitCode()) + .executeUpdate(); + } + + Map result = new LinkedHashMap<>(); + String instruction = toStr(prescription.get("instruction")); + if (!drugs.isEmpty() || (instruction != null && !instruction.isBlank())) { + long prescriptionID = insertPrescription(beneficiaryRegID, visit, instruction, createdBy, vanID, + parkingPlaceID); + for (Map d : drugs) { + Integer qty = toInt(d.get("qtyPrescribed")); + if (qty == null || qty <= 0) qty = calculateQuantity(d, toInt(d.get("itemFormID"))); // null if unknown + em.createNativeQuery("INSERT INTO db_iemr.t_prescribeddrug (BeneficiaryRegID, BenVisitID, " + + "ProviderServiceMapID, VisitCode, PrescriptionID, DrugForm, DrugID, GenericDrugName, Dose, " + + "Frequency, Duration, DuartionUnit, SpecialInstruction, QtyPrescribed, Deleted, Processed, " + + "CreatedBy, VanID, ParkingPlaceID) VALUES (:ben, :visitID, :psm, :visitCode, " + + ":prescriptionID, :form, :drugID, :name, :dose, :frequency, :duration, :durationUnit, " + + ":instruction, :qty, false, 'N', :user, :van, :pp)") + .setParameter("ben", beneficiaryRegID) + .setParameter("visitID", visit.getBenVisitId()) + .setParameter("psm", visit.getProviderServiceMapID()) + .setParameter("visitCode", visit.getVisitCode()) + .setParameter("prescriptionID", prescriptionID) + .setParameter("form", toStr(d.get("drugForm"))) + .setParameter("drugID", toInt(d.get("drugID"))) + .setParameter("name", toStr(d.get("drugName"))) + .setParameter("dose", toStr(d.get("dose"))) + .setParameter("frequency", toStr(d.get("frequency"))) + .setParameter("duration", toStr(d.get("duration"))) + .setParameter("durationUnit", toStr(d.get("durationUnit"))) + .setParameter("instruction", toStr(d.get("instructions"))) + .setParameter("qty", qty) + .setParameter("user", createdBy) + .setParameter("van", vanID) + .setParameter("pp", parkingPlaceID) + .executeUpdate(); + stampVanSerialNo("t_prescribeddrug", "PrescribedDrugID", lastInsertId()); + } + result.put("prescriptionID", prescriptionID); + } + result.put("dispensed", new ArrayList>()); + return result; + } + // t_prescription row for the visit (Stop TB service line, same as the visit). private long insertPrescription(Long beneficiaryRegID, BenVisitDetail visit, String instruction, String createdBy, Integer vanID, Integer parkingPlaceID) { @@ -508,8 +580,7 @@ private long insertPrescription(Long beneficiaryRegID, BenVisitDetail visit, Str * one intake per 7 days. Other forms (syrup, cream, injection, drops...): 1 bottle/tube/vial. Null when a * tablet/capsule quantity cannot be worked out (unknown frequency or missing duration). */ - private Integer calculateQuantity(Map drug, Object[] item) { - Integer formID = item[7] != null ? ((Number) item[7]).intValue() : null; + private Integer calculateQuantity(Map drug, Integer formID) { if (formID == null || (formID != 1 && formID != 2)) return 1; // 1 = Tablet, 2 = Capsule double perIntake = formID == 2 ? 1 : tabletsPerIntake(toStr(drug.get("dose"))); String frequency = Optional.ofNullable(toStr(drug.get("frequency"))).orElse("").toUpperCase();