feat: add two reprocess endpoints

.github/workflows/docker.yaml

@@ -3,7 +3,8 @@ name: Build snapshot docker image
 on:
   push:
     branches-ignore:
-      - main
+      - * # main
+      # turned off for now (needs secrets renewal)
 
 jobs:
   build-snapshot:

pom.xml

@@ -36,7 +36,7 @@
         <!-- Pi Test-->
         <pitest.version>1.22.1</pitest.version>
         <pitest.junit.version>1.2.3</pitest.junit.version>
-        <bpm.version>1.1.0</bpm.version>
+        <bpm.version>1.1.1</bpm.version>
     </properties>
     <dependencies>
         <dependency>

src/main/java/fr/insee/genesis/controller/exception/ExceptionController.java

@@ -0,0 +1,44 @@
+package fr.insee.genesis.controller.exception;
+
+import fr.insee.genesis.exceptions.GenesisException;
+import fr.insee.genesis.exceptions.InvalidDateIntervalException;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ProblemDetail;
+import org.springframework.web.bind.annotation.ControllerAdvice;
+import org.springframework.web.bind.annotation.ExceptionHandler;
+
+/**
+ * This controller uses Spring's ControllerAdvice annotation to intercept exceptions.
+ * It implements the <a href="https://www.rfc-editor.org/rfc/rfc9457.html">RFC 9457</a> by returning
+ * Spring's <code>ProblemDetail</code> object.
+ */
+@ControllerAdvice
+@Slf4j
+public class ExceptionController {
+
+    // Note: No handler for uncaught Exception.class for now since it breaks soms tests.
+
+    @ExceptionHandler
+    public ProblemDetail handleGenericGenesisException(GenesisException genesisException) {
+        log.error("GenesisException: {}", genesisException.getMessage(), genesisException);
+        return ProblemDetail.forStatusAndDetail(
+                resolveHttpCode(genesisException.getStatus()),
+                genesisException.getMessage());
+    }
+
+    /** Returns the corresponding http status, or 500 if the given code does not match a http status. */
+    private static HttpStatus resolveHttpCode(int statusCode) {
+        HttpStatus httpStatus = HttpStatus.resolve(statusCode);
+        return httpStatus != null ? httpStatus : HttpStatus.INTERNAL_SERVER_ERROR;
+    }
+
+    @ExceptionHandler(InvalidDateIntervalException.class)
+    public ProblemDetail handleInvalidDateIntervalException(InvalidDateIntervalException e) {
+        log.error("InvalidDateIntervalException: {}", e.getMessage());
+        return ProblemDetail.forStatusAndDetail(
+                HttpStatus.BAD_REQUEST,
+                e.getMessage());
+    }
+
+}

src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java

@@ -15,6 +15,7 @@
 import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.Parameter;
 import jakarta.validation.Valid;
+import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.PageRequest;
@@ -43,23 +44,16 @@
 
 @Slf4j
 @Controller
+@RequiredArgsConstructor
 public class RawResponseController {
 
     private static final String SUCCESS_MESSAGE = "Interrogation %s saved";
     private static final String INTERROGATION_ID = "interrogationId";
-    public static final String NB_DOCS_WITH_FORMATTED = "%d document(s) processed, including %d FORMATTED after data verification for collectionInstrumentId %s";
-    public static final String NB_DOCS = "%d document(s) processed for collectionInstrumentId %s";
+
     private final LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort;
     private final RawResponseApiPort rawResponseApiPort;
     private final RawResponseInputRepository rawRepository;
 
-
-    public RawResponseController(LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort, RawResponseApiPort rawResponseApiPort, RawResponseInputRepository rawRepository) {
-        this.lunaticJsonRawDataApiPort = lunaticJsonRawDataApiPort;
-        this.rawResponseApiPort = rawResponseApiPort;
-        this.rawRepository = rawRepository;
-    }
-
     @Operation(summary = "Save lunatic json data from one interrogation in Genesis Database")
     @PutMapping(path = "/responses/raw/lunatic-json/save")
     @PreAuthorize("hasRole('COLLECT_PLATFORM')")
@@ -119,10 +113,7 @@ public ResponseEntity<String> processRawResponses(
         List<GenesisError> errors = new ArrayList<>();
         try {
             DataProcessResult result = rawResponseApiPort.processRawResponses(collectionInstrumentId, interrogationIdList, errors);
-            return result.formattedDataCount() == 0 ?
-                    ResponseEntity.ok(NB_DOCS.formatted(result.dataCount(), collectionInstrumentId))
-                    : ResponseEntity.ok(NB_DOCS_WITH_FORMATTED
-                    .formatted(result.dataCount(), result.formattedDataCount(), collectionInstrumentId));
+            return ResponseEntity.ok(result.message(collectionInstrumentId));
         } catch (GenesisException e) {
             return ResponseEntity.status(e.getStatus()).body(e.getMessage());
         }
@@ -141,10 +132,7 @@ public ResponseEntity<String> processRawResponsesByCollectionInstrumentId(
         log.info("Try to process raw responses for collectionInstrumentId {}", collectionInstrumentId);
         try {
             DataProcessResult result = rawResponseApiPort.processRawResponses(collectionInstrumentId);
-            return result.formattedDataCount() == 0 ?
-                    ResponseEntity.ok(NB_DOCS.formatted(result.dataCount(), collectionInstrumentId))
-                    : ResponseEntity.ok(NB_DOCS_WITH_FORMATTED
-                    .formatted(result.dataCount(), result.formattedDataCount(), collectionInstrumentId));
+            return ResponseEntity.ok(result.message(collectionInstrumentId));
         } catch (GenesisException e) {
             return ResponseEntity.status(e.getStatus()).body(e.getMessage());
         }
@@ -183,7 +171,7 @@ public ResponseEntity<LunaticJsonRawDataModel> getJsonRawData(
             @RequestParam("campaignName") String campaignName,
             @RequestParam(value = "mode") Mode modeSpecified
     ) {
-        List<LunaticJsonRawDataModel> data = lunaticJsonRawDataApiPort.getRawData(campaignName, modeSpecified, List.of(interrogationId));
+        List<LunaticJsonRawDataModel> data = lunaticJsonRawDataApiPort.getRawDataByQuestionnaireId(campaignName, modeSpecified, List.of(interrogationId));
         return ResponseEntity.ok(data.getFirst());
     }
 

src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseReprocessController.java

@@ -0,0 +1,94 @@
+package fr.insee.genesis.controller.rest.responses;
+
+import fr.insee.genesis.domain.model.surveyunit.rawdata.DataProcessResult;
+import fr.insee.genesis.domain.model.surveyunit.rawdata.RawDataModelType;
+import fr.insee.genesis.domain.ports.api.ReprocessRawResponseApiPort;
+import fr.insee.genesis.exceptions.GenesisException;
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.Parameter;
+import io.swagger.v3.oas.annotations.media.Schema;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.format.annotation.DateTimeFormat;
+import org.springframework.http.ResponseEntity;
+import org.springframework.security.access.prepost.PreAuthorize;
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+
+import java.time.LocalDateTime;
+
+@Controller
+@RequiredArgsConstructor
+@Slf4j
+public class RawResponseReprocessController {
+
+    private final ReprocessRawResponseApiPort reprocessRawResponseApiPort;
+
+    @Operation(summary = "Reprocess raw response of a collection instrument.")
+    @PostMapping(path = "/raw-responses/{collectionInstrumentId}/reprocess")
+    @PreAuthorize("hasRole('ADMIN')")
+    public ResponseEntity<String> reProcessRawResponsesByCollectionInstrumentId(
+            @Parameter(
+                    description = "Id of the collection instrument (old questionnaireId)",
+                    example = "ENQTEST2025X00")
+            @PathVariable("collectionInstrumentId")
+            String collectionInstrumentId,
+
+            @Parameter(description = "Extract since",
+                    schema = @Schema(type = "string", format = "date-time", example = "2026-01-01T00:00:00"))
+            @RequestParam(value = "sinceDate", required = false)
+            @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME)
+            LocalDateTime sinceDate,
+
+            @Parameter(description = "Extract until",
+                    schema = @Schema(type = "string", format = "date-time", example = "2026-02-02T00:00:00"))
+            @RequestParam(value = "endDate", required = false)
+            @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME)
+            LocalDateTime endDate
+    ) throws GenesisException {
+
+        DataProcessResult result = reprocessRawResponseApiPort.reprocessRawResponses(
+                RawDataModelType.FILIERE,
+                collectionInstrumentId,
+                sinceDate,
+                endDate);
+
+        return ResponseEntity.ok(result.message(collectionInstrumentId));
+    }
+
+    @Operation(summary = "Reprocess Lunatic raw data for a questionnaire model. " +
+            "**Note**: Lunatic raw data is the legacy format of raw responses.")
+    @PostMapping(path = "/responses/raw/lunatic-json/{questionnaireId}/reprocess")
+    @PreAuthorize("hasRole('ADMIN')")
+    public ResponseEntity<String> reProcessJsonRawDataByQuestionnaireId(
+            @Parameter(
+                    description = "Questionnaire model id (old name for collection instrument id).",
+                    example = "ENQTEST2025X00")
+            @PathVariable("questionnaireId")
+            String collectionInstrumentId, // 'questionnaireId' is the legacy name for 'collectionInstrumentId'
+
+            @Parameter(description = "Extract since",
+                    schema = @Schema(type = "string", format = "date-time", example = "2026-01-01T00:00:00"))
+            @RequestParam(value = "sinceDate", required = false)
+            @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME)
+            LocalDateTime sinceDate,
+
+            @Parameter(description = "Extract until",
+                    schema = @Schema(type = "string", format = "date-time", example = "2026-02-02T00:00:00"))
+            @RequestParam(value = "endDate", required = false)
+            @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME)
+            LocalDateTime endDate
+    ) throws GenesisException {
+
+        DataProcessResult result = reprocessRawResponseApiPort.reprocessRawResponses(
+                RawDataModelType.LEGACY,
+                collectionInstrumentId,
+                sinceDate,
+                endDate);
+
+        return ResponseEntity.ok(result.message(collectionInstrumentId));
+    }
+
+}

src/main/java/fr/insee/genesis/controller/utils/ControllerUtils.java

@@ -26,19 +26,19 @@ public ControllerUtils(FileUtils fileUtils) {
 
 	/**
 	 * If a mode is specified, we treat only this mode.
-	 * If no mode is specified, we treat all modes in the campaign.
+	 * If no mode is specified, we treat all modes in the questionnaireId.
 	 * If no mode is specified and no specs are found, we return an error
-	 * @param campaign campaign id to get modes
+	 * @param questionnaireId questionnaireId id to get modes
 	 * @param modeSpecified a Mode to use, null if we want all modes available
 	 * @return a list with the mode in modeSpecified or all modes if null
 	 * @throws GenesisException if error in specs structure
 	 */
-	public List<Mode> getModesList(String campaign, Mode modeSpecified) throws GenesisException {
+	public List<Mode> getModesList(String questionnaireId, Mode modeSpecified) throws GenesisException {
 		if (modeSpecified != null){
 			return Collections.singletonList(modeSpecified);
 		}
 		List<Mode> modes = new ArrayList<>();
-		String specFolder = fileUtils.getSpecFolder(campaign);
+		String specFolder = fileUtils.getSpecFolder(questionnaireId);
 		List<String> modeSpecFolders = fileUtils.listFolders(specFolder);
 		if (modeSpecFolders.isEmpty()) {
 			throw new GenesisException(404, "No specification folder found " + specFolder);

src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/DataProcessResult.java

@@ -4,5 +4,31 @@
 
 import java.util.List;
 
-public record DataProcessResult(int dataCount, int formattedDataCount, List<GenesisError> errors) {
+public record DataProcessResult(
+        int dataCount,
+        int formattedDataCount,
+        List<GenesisError> errors) {
+
+    public String message(String collectionInstrumentId) {
+        return String.format("%s%s%s.",
+                interrogationCountMessage(dataCount),
+                formattedCountMessage(formattedDataCount),
+                collectionIdMessage(collectionInstrumentId));
+    }
+
+    private static String interrogationCountMessage(int processedInterrogationsCount) {
+        boolean plural = processedInterrogationsCount > 1;
+        return "%d interrogation%s processed".formatted(processedInterrogationsCount,  plural ? "s" : "");
+    }
+
+    private static String collectionIdMessage(String collectionInstrumentId) {
+        return " for collectionInstrumentId '%s'".formatted(collectionInstrumentId);
+    }
+
+    private static String formattedCountMessage(int formattedCount) {
+        if (formattedCount == 0)
+            return "";
+        return " (including %d FORMATTED after data verification)".formatted(formattedCount);
+    }
+
 }

src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/RawDataModelType.java

@@ -1,5 +1,20 @@
 package fr.insee.genesis.domain.model.surveyunit.rawdata;
 
+/** Format of raw data to be imported into the data storage. */
 public enum RawDataModelType {
-    DEFAULT, FILIERE
+
+    /** Legacy format of raw data ('Lunatic'). */
+    LEGACY,
+
+    /** 'Filière' raw response model. */
+    FILIERE;
+
+    @Override
+    public String toString() {
+        return switch (this) {
+            case LEGACY -> "LEGACY (Lunatic)";
+            case FILIERE -> "FILIERE raw responses";
+        };
+    }
+
 }

src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java

@@ -11,18 +11,18 @@
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.Pageable;
 
-import java.time.LocalDateTime;
 import java.time.Instant;
+import java.time.LocalDateTime;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
 public interface LunaticJsonRawDataApiPort {
 
     void save(LunaticJsonRawDataModel rawData);
-    List<LunaticJsonRawDataModel> getRawData(String campaignName, Mode mode, List<String> interrogationIdList);
     List<LunaticJsonRawDataModel> getRawDataByQuestionnaireId(String questionnaireId, Mode mode, List<String> interrogationIdList);
     List<SurveyUnitModel> convertRawData(List<LunaticJsonRawDataModel> rawData, VariablesMap variablesMap);
+
     List<LunaticJsonRawDataUnprocessedDto> getUnprocessedDataIds();
     Set<String> getUnprocessedDataQuestionnaireIds();
     void updateProcessDates(List<SurveyUnitModel> surveyUnitModels);
@@ -33,6 +33,9 @@ public interface LunaticJsonRawDataApiPort {
 
     List<LunaticJsonRawDataModel> getRawDataByInterrogationId(String interrogationId);
 
+    /**
+     * @deprecated Use the method with 'collectionInstrumentId' instead.
+     */
     @Deprecated(since = "1.13.0")
     DataProcessResult processRawData(String campaignName, List<String> interrogationIdList, List<GenesisError> errors) throws GenesisException;
 

src/main/java/fr/insee/genesis/domain/ports/api/RawResponseApiPort.java

@@ -20,6 +20,7 @@ public interface RawResponseApiPort {
     List<RawResponseModel> getRawResponsesByInterrogationID(String interrogationId);
     DataProcessResult processRawResponses(String collectionInstrumentId, List<String> interrogationIdList, List<GenesisError> errors) throws GenesisException;
     DataProcessResult processRawResponses(String collectionInstrumentId) throws GenesisException;
+
     List<SurveyUnitModel> convertRawResponse(List<RawResponseModel> rawResponses, VariablesMap variablesMap);
     List<String> getUnprocessedCollectionInstrumentIds();
     void updateProcessDates(List<SurveyUnitModel> surveyUnitModels);

src/main/java/fr/insee/genesis/domain/ports/api/ReprocessRawResponseApiPort.java

@@ -0,0 +1,26 @@
+package fr.insee.genesis.domain.ports.api;
+
+import fr.insee.genesis.domain.model.surveyunit.rawdata.DataProcessResult;
+import fr.insee.genesis.domain.model.surveyunit.rawdata.RawDataModelType;
+import fr.insee.genesis.exceptions.GenesisException;
+
+import java.time.LocalDateTime;
+
+public interface ReprocessRawResponseApiPort {
+
+    /**
+     * Reprocesses raw data of the collection that correspond to the given identifier.
+     * An optional date interval can be given to reprocess a subset of the collection.
+     * @param rawDataModelType {@link RawDataModelType}
+     * @param collectionInstrumentId Collection instrument identifier.
+     * @param sinceDate Start of the date interval.
+     * @param endDate End of the date interval.
+     * @return Data processing result record.
+     * @see DataProcessResult
+     */
+    DataProcessResult reprocessRawResponses(
+            RawDataModelType rawDataModelType,
+            String collectionInstrumentId, LocalDateTime sinceDate, LocalDateTime endDate)
+            throws GenesisException;
+
+}

src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java

@@ -1,11 +1,7 @@
 package fr.insee.genesis.domain.ports.api;
 
 import fr.insee.bpm.metadata.model.VariablesMap;
-import fr.insee.genesis.controller.dto.CampaignWithQuestionnaire;
-import fr.insee.genesis.controller.dto.QuestionnaireWithCampaign;
-import fr.insee.genesis.controller.dto.SurveyUnitDto;
-import fr.insee.genesis.controller.dto.SurveyUnitInputDto;
-import fr.insee.genesis.controller.dto.SurveyUnitSimplifiedDto;
+import fr.insee.genesis.controller.dto.*;
 import fr.insee.genesis.domain.model.surveyunit.InterrogationId;
 import fr.insee.genesis.domain.model.surveyunit.Mode;
 import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel;
@@ -73,6 +69,11 @@ List<InterrogationId> findDistinctPageableInterrogationIdsByQuestionnaireId(Stri
 
     Long deleteByCollectionInstrumentId(String collectionInstrumentId);
 
+    Long deleteByQuestionnaireIdAndInterrogationIds(
+            String questionnaireId,
+            Set<String> interrogationIds
+    );
+
     long countResponses();
 
     Set<String> findQuestionnaireIdsByCampaignId(String campaignId);