Merge branch 'DEVELOP' of...

Merge branch 'DEVELOP' of ssh://git.eclipse.org:29418/openk-usermodules/org.eclipse.openk-usermodules.gridFailureInformation.backend into SI-381-Auswahl-aus-Adressbestand

addressImport/Dockerfile

+FROM openjdk:8-jre-alpine
+COPY /target/grid-failure-information.addressimport.service.jar /usr/src/cbd/
+WORKDIR usr/src/cbd
+CMD ["java", "-jar", "-Dspring.profiles.active=devserver", "grid-failure-information.addressimport.service.jar"]
\ No newline at end of file

addressImport/lombok.config

+config.stopBubbling = true
+lombok.addLombokGeneratedAnnotation = true
\ No newline at end of file

addressImport/pom.xml

@@ -9,10 +9,10 @@
 		<relativePath/> <!-- lookup parent from repository -->
 	</parent>
 	<groupId>org.eclipse.openk</groupId>
-	<artifactId>grid-failure-information.backend.service</artifactId>
+	<artifactId>grid-failure-information.addressimport.service</artifactId>
 	<version>0.0.1-SNAPSHOT</version>
 	<name>grid-failure-information</name>
-	<description>Grid-Failure-Information Project for openKONSEQUENZ</description>
+	<description>Address Import of Grid-Failure-Information Project for openKONSEQUENZ</description>
 
 	<properties>
 		<maven.test.skip>false</maven.test.skip>
@@ -26,6 +26,15 @@
 		<jruby-complete-version>9.0.0.0</jruby-complete-version>
 		<mapstruct.version>1.2.0.Final</mapstruct.version>
 		<flyway-core.version>6.0.8</flyway-core.version>
+		
+		<flyway.sqlMigrationSeparator>__</flyway.sqlMigrationSeparator>
+		<flyway.url>jdbc:postgresql://entopticadirx:5432/GridFailureInfoDevServer</flyway.url>
+		<flyway.user>gfi_service</flyway.user>
+		<flyway.password>gfi_service</flyway.password>
+		<flyway.schemas>public</flyway.schemas>
+		<flyway.table>address_flyway_schema_history</flyway.table>
+		<flyway.locations>classpath:db/migration/</flyway.locations>
+
 		<postgresql.version>42.2.8</postgresql.version>
 		<lombock.version>1.18.10</lombock.version>
 		<h2.version>1.4.200</h2.version>

addressImport/src/main/java/org/eclipse/openk/gridfailureinformation/importadresses/AddressImportApplication.java

@@ -9,7 +9,9 @@ import springfox.documentation.swagger2.annotations.EnableSwagger2;
 public class AddressImportApplication {
 
 	public static void main(String[] args) {
-		SpringApplication.run(AddressImportApplication.class, args);
+		// passing through args is a security issue at sonar. As long as we don't know what for, we won't passthrough
+		// any arg
+		SpringApplication.run(AddressImportApplication.class, new String[0]);
 	}
 
 }

addressImport/src/main/java/org/eclipse/openk/gridfailureinformation/importadresses/controller/AdressImportController.java

@@ -4,11 +4,10 @@ import io.swagger.annotations.ApiOperation;
 import io.swagger.annotations.ApiResponse;
 import io.swagger.annotations.ApiResponses;
 import lombok.extern.log4j.Log4j2;
-import org.eclipse.openk.gridfailureinformation.importadresses.service.AddressImportService;
+import org.eclipse.openk.gridfailureinformation.importadresses.jobs.JobManager;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
 
 
@@ -18,17 +17,16 @@ import org.springframework.web.bind.annotation.RestController;
 public class AdressImportController {
 
     @Autowired
-    private AddressImportService addressImportService;
+    private JobManager jobManager;
+
 
-    @PostMapping
     @ApiOperation(value = "Import von Adressen und Stromstationen")
     @ApiResponses(value = {
             @ApiResponse(code = 201, message = "Adressen erfolgreich importiert"),
             @ApiResponse(code = 500, message = "Konnte nicht durchgeführt werden")
     })
-    @RequestMapping("/import")
-    public void startImportAddresses(
-            @RequestParam( value="cleanup", required = false,  defaultValue = "true") boolean cleanup ) {
-        addressImportService.importAddresses(cleanup);
+    @PostMapping("/import")
+    public void startImportAddresses() {
+        jobManager.triggerStartImport();
     }
 }
\ No newline at end of file

addressImport/src/main/java/org/eclipse/openk/gridfailureinformation/importadresses/exceptions/InternalServerErrorException.java

+/*
+ *******************************************************************************
+ * Copyright (c) 2019 Contributors to the Eclipse Foundation
+ *
+ * See the NOTICE file(s) distributed with this work for additional
+ * information regarding copyright ownership.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v. 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *******************************************************************************
+ */
+package org.eclipse.openk.gridfailureinformation.importadresses.exceptions;
+
+import org.springframework.http.HttpStatus;
+import org.springframework.web.bind.annotation.ResponseStatus;
+
+@ResponseStatus(code = HttpStatus.INTERNAL_SERVER_ERROR)
+public class InternalServerErrorException extends RuntimeException {
+    public InternalServerErrorException(String message) {
+        super(message);
+    }
+}

addressImport/src/main/java/org/eclipse/openk/gridfailureinformation/importadresses/jobs/JobManager.java

@@ -4,10 +4,16 @@ import lombok.extern.log4j.Log4j2;
 import org.eclipse.openk.gridfailureinformation.importadresses.service.AddressImportService;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
+import org.springframework.http.HttpStatus;
 import org.springframework.scheduling.annotation.EnableScheduling;
 import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Component;
 
+import javax.xml.ws.http.HTTPException;
+
+import static java.lang.Boolean.FALSE;
+import static java.lang.Boolean.TRUE;
+
 @EnableScheduling
 @Component
 @Log4j2
@@ -18,18 +24,45 @@ public class JobManager {
 
     @Autowired
     AddressImportService addressImportService;
+    private final Object lock = new Object();
+
+    private Boolean startImport = FALSE;
 
-    // job for impoerting address and station data
-//    @Scheduled(initialDelay = 1000, fixedRate = 900000000)
-//    public void importAddressdata() {
-//            boolean cleanUp = true;
-//            addressImportService.importAddresses(true);
-//        }
-//
-//    }
+    public void triggerStartImport() {
+        synchronized (lock) {
+            startImport = TRUE;
+        }
+    }
+
+    @Scheduled(cron="0/3 0/1 * 1/1 * *") // every 3 seconds
+    private void importOnTrigger() {
+        boolean go = false;
+
+        synchronized (lock) {
+            if( startImport.equals(TRUE) ) {
+                go = true;
+                startImport = FALSE;
+            }
+        }
+
+        if( go ) {
+            importAddressdata();
+        }
+
+    }
 
     @Scheduled(cron= "${adressimport.cron}")
-    public void importAddressdata() {
+    private void importAddressdata() {
+        try {
             addressImportService.importAddresses(cleanUp);
         }
+        catch( HTTPException e ) {
+            if( e.getStatusCode() != HttpStatus.PROCESSING.value() ) {
+                throw( e );
+            }
+            else {
+                log.info("Jobmanager triggered import, but import already in work");
+            }
+        }
+    }
 }

addressImport/src/main/java/org/eclipse/openk/gridfailureinformation/importadresses/service/AddressImportService.java

@@ -11,11 +11,15 @@ import org.eclipse.openk.gridfailureinformation.importadresses.viewmodel.Address
 import org.eclipse.openk.gridfailureinformation.importadresses.viewmodel.StationDto;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
+import org.springframework.http.HttpStatus;
+import org.springframework.scheduling.annotation.Async;
 import org.springframework.stereotype.Service;
-
+import javax.xml.ws.http.HTTPException;
 import java.io.FileReader;
 import java.io.IOException;
 import java.math.BigDecimal;
+import java.time.Duration;
+import java.time.Instant;
 import java.util.List;
 
 @Service
@@ -42,32 +46,68 @@ public class AddressImportService {
     @Value("${adressimport.file.power-stations}")
     public String filePowerStations;
 
-    public void importAddresses(Boolean cleanUp) {
+    private final Object lock = new Object();
+    private Boolean importInWork=Boolean.FALSE;
+
+    @Async
+    public void importAddresses(boolean cleanUp) {
+        // Don't import twice
+        synchronized (lock) {
+            if(importInWork.equals(Boolean.TRUE)) {
+                throw new HTTPException(HttpStatus.PROCESSING.value());
+            }
+            else {
+                importInWork = Boolean.TRUE;
+            }
+        }
 
         if (cleanUp) {
+            Instant cleanUpStart = Instant.now();
             log.info("CleanUp START");
+
             addressService.deleteAllAddresses();
             stationService.deleteAllStations();
+
             log.info("CleanUp ENDE");
+            Instant cleanUpEnd = Instant.now();
+            log.info("CleanUp Duration: " + Duration.between(cleanUpStart, cleanUpEnd));
         }
 
-        log.info("Adressimport START");
+        Instant importStart = Instant.now();
+        log.info("Import START");
+
+        log.info("- Import Addresses START");
         importAddressFile(cleanUp);
-        log.info("Import Power Connections START");
+        log.info("- Import Addresses END");
+
+        log.info("- Import Power Connections START");
         importPowerConnectionsAddressFile();
-        log.info("Import Water Connections START");
+        log.info("- Import Power Connections END");
+
+        log.info("- Import Water Connections START");
         importWaterConnectionsAddressFile();
-        log.info("Import Gas Connections START");
+        log.info("- Import Water Connections END");
+
+        log.info("- Import Gas Connections START");
         importGasConnectionsAddressFile();
-        log.info("Adressimport ENDE");
-        log.info("Import Power Stations START");
+        log.info("- Import Gas Connections END");
+
+        log.info("- Import PowerStations START");
         importPowerStationFile(cleanUp);
-        log.info("Stationsimport ENDE");
+        log.info("- Import PowerStations END");
+
+        log.info("Import END");
+        Instant importEnd = Instant.now();
+        log.info("Total Import Duration: " + Duration.between(importStart, importEnd));
+
+        synchronized (lock) {
+            importInWork = Boolean.FALSE;
+        }
     }
 
 
     public void importAddressFile(boolean isTableClean) {
-
+        Instant start = Instant.now();
         String csvFile = fileAddresses;
 
         CSVReader reader = null;
@@ -81,49 +121,42 @@ public class AddressImportService {
                     .build();
             String[]line;
             while ((line = reader.readNext()) != null) {
-                AddressDto addressDto = new AddressDto();
-                // !!! Parsing BigDecimal  anpassen, locale
-                BigDecimal x1 = new BigDecimal(line[0].replaceAll(",", "."));
-                addressDto.setSdox1(new BigDecimal(line[0].replaceAll(",", ".")));
-                addressDto.setSdoy1(new BigDecimal(line[1].replaceAll(",", ".")));
-                addressDto.setG3efid(Long.parseLong(line[2]));
-                addressDto.setPostcode(line[3]);
-                addressDto.setCommunity(line[4]);
-                addressDto.setDistrict(line[5]);
-                addressDto.setStreet(line[6]);
-                addressDto.setHousenumber(line[7]);
-
-                List<BigDecimal> decimals = converter.convertUTMToDeg(addressDto.getSdox1().doubleValue(), addressDto.getSdoy1().doubleValue());
-                addressDto.setLongitude(decimals.get(0));
-                addressDto.setLatitude(decimals.get(1));
+                storeAddress(isTableClean, line);
 
-                AddressDto savedAddressDto;
-                if (isTableClean) {
-                    savedAddressDto = addressService.insertAddress(addressDto);
-                } else {
-                    savedAddressDto = addressService.updateOrInsertAddressByG3efid(addressDto);
-                }
-
-               /* System.out.println("Adresse [sdo_x1= " + line[0]
-                        + ", sdo_y1= " + line[1]
-                        + ", g3e_fid=" + line[2]
-                        + ", long= " + addressDto.getLongitude()
-                        + ", lat= " + addressDto.getLatitude()
-                        + ", plz=" + line[3]
-                        + ", ort=" + line[4]
-                        + ", ortsteil=" + line[5]
-                        + ", strasse=" + line[6]
-                        + ", hausnummer=" + line[7]
-                        + "]");*/
             }
         } catch (IOException | CsvValidationException e) {
-            e.printStackTrace();
+            log.error(e);
         }
+        Instant end = Instant.now();
+        log.info("Duration: " + Duration.between(start, end));
     }
 
+    private void storeAddress(boolean isTableClean, String[] line) {
+        AddressDto addressDto = new AddressDto();
+        // !!! Parsing BigDecimal  anpassen, locale
+        addressDto.setSdox1(new BigDecimal(line[0].replace(",", ".")));
+        addressDto.setSdoy1(new BigDecimal(line[1].replace(",", ".")));
+        addressDto.setG3efid(Long.parseLong(line[2]));
+        addressDto.setPostcode(line[3]);
+        addressDto.setCommunity(line[4]);
+        addressDto.setDistrict(line[5]);
+        addressDto.setStreet(line[6]);
+        addressDto.setHousenumber(line[7]);
+
+        List<BigDecimal> decimals = converter.convertUTMToDeg(addressDto.getSdox1().doubleValue(), addressDto.getSdoy1().doubleValue());
+        addressDto.setLongitude(decimals.get(0));
+        addressDto.setLatitude(decimals.get(1));
+
+        if (isTableClean) {
+            addressService.insertAddress(addressDto);
+        } else {
+            addressService.updateOrInsertAddressByG3efid(addressDto);
+        }
+    }
 
-    public void importPowerConnectionsAddressFile() {
 
+    public void importPowerConnectionsAddressFile() {
+        Instant start = Instant.now();
         String csvFile = filePowerConnections;
 
         CSVReader reader = null;
@@ -139,9 +172,8 @@ public class AddressImportService {
             while ((line = reader.readNext()) != null) {
                 AddressDto addressDto = new AddressDto();
                 // !!! Parsing BigDecimal  anpassen, locale???
-                BigDecimal x1 = new BigDecimal(line[0].replaceAll(",", "."));
-                addressDto.setSdox1(new BigDecimal(line[0].replaceAll(",", ".")));
-                addressDto.setSdoy1(new BigDecimal(line[1].replaceAll(",", ".")));
+                addressDto.setSdox1(new BigDecimal(line[0].replace(",", ".")));
+                addressDto.setSdoy1(new BigDecimal(line[1].replace(",", ".")));
                 addressDto.setG3efid(Long.parseLong(line[2]));
                 addressDto.setPostcode(line[3]);
                 addressDto.setCommunity(line[4]);
@@ -155,16 +187,18 @@ public class AddressImportService {
                 addressDto.setLongitude(decimals.get(0));
                 addressDto.setLatitude(decimals.get(1));
 
-                AddressDto savedAddressDto = addressService.updateOrInsertAddressByG3efid(addressDto);
+                addressService.updateOrInsertAddressByG3efid(addressDto);
             }
         } catch (IOException | CsvValidationException e) {
-            e.printStackTrace();
+            log.error(e);
         }
+        Instant end = Instant.now();
+        log.info("Duration: " + Duration.between(start, end));
     }
 
 
     public void importWaterConnectionsAddressFile() {
-
+        Instant start = Instant.now();
         String csvFile = fileWaterConnections;
 
         CSVReader reader = null;
@@ -180,9 +214,8 @@ public class AddressImportService {
             while ((line = reader.readNext()) != null) {
                 AddressDto addressDto = new AddressDto();
                 // !!! Parsing BigDecimal  anpassen, locale???
-                BigDecimal x1 = new BigDecimal(line[0].replaceAll(",", "."));
-                addressDto.setSdox1(new BigDecimal(line[0].replaceAll(",", ".")));
-                addressDto.setSdoy1(new BigDecimal(line[1].replaceAll(",", ".")));
+                addressDto.setSdox1(new BigDecimal(line[0].replace(",", ".")));
+                addressDto.setSdoy1(new BigDecimal(line[1].replace(",", ".")));
                 addressDto.setG3efid(Long.parseLong(line[2]));
                 addressDto.setPostcode(line[3]);
                 addressDto.setCommunity(line[4]);
@@ -196,15 +229,17 @@ public class AddressImportService {
                 addressDto.setLongitude(decimals.get(0));
                 addressDto.setLatitude(decimals.get(1));
 
-                AddressDto savedAddressDto = addressService.updateOrInsertAddressByG3efid(addressDto);
+                addressService.updateOrInsertAddressByG3efid(addressDto);
             }
         } catch (IOException | CsvValidationException e) {
-            e.printStackTrace();
+            log.error(e);
         }
+        Instant end = Instant.now();
+        log.info("Duration: " + Duration.between(start, end));
     }
 
     public void importGasConnectionsAddressFile() {
-
+        Instant start = Instant.now();
         String csvFile = fileGasConnections;
 
         CSVReader reader = null;
@@ -220,9 +255,8 @@ public class AddressImportService {
             while ((line = reader.readNext()) != null) {
                 AddressDto addressDto = new AddressDto();
                 // !!! Parsing BigDecimal  anpassen, locale???
-                BigDecimal x1 = new BigDecimal(line[0].replaceAll(",", "."));
-                addressDto.setSdox1(new BigDecimal(line[0].replaceAll(",", ".")));
-                addressDto.setSdoy1(new BigDecimal(line[1].replaceAll(",", ".")));
+                addressDto.setSdox1(new BigDecimal(line[0].replace(",", ".")));
+                addressDto.setSdoy1(new BigDecimal(line[1].replace(",", ".")));
                 addressDto.setG3efid(Long.parseLong(line[2]));
                 addressDto.setPostcode(line[3]);
                 addressDto.setCommunity(line[4]);
@@ -236,15 +270,17 @@ public class AddressImportService {
                 addressDto.setLongitude(decimals.get(0));
                 addressDto.setLatitude(decimals.get(1));
 
-                AddressDto savedAddressDto = addressService.updateOrInsertAddressByG3efid(addressDto);
+                addressService.updateOrInsertAddressByG3efid(addressDto);
             }
         } catch (IOException | CsvValidationException e) {
-            e.printStackTrace();
+            log.error(e);
         }
+        Instant end = Instant.now();
+        log.info("Duration: " + Duration.between(start, end));
     }
 
     public void importPowerStationFile(boolean isTableClean) {
-
+        Instant start = Instant.now();
         String csvFile = filePowerStations;
 
         CSVReader reader = null;
@@ -260,9 +296,8 @@ public class AddressImportService {
             while ((line = reader.readNext()) != null) {
                 StationDto stationDto = new StationDto();
                 // !!! Parsing BigDecimal  anpassen, locale
-                BigDecimal x1 = new BigDecimal(line[0].replaceAll(",", "."));
-                stationDto.setSdox1(new BigDecimal(line[0].replaceAll(",", ".")));
-                stationDto.setSdoy1(new BigDecimal(line[1].replaceAll(",", ".")));
+                stationDto.setSdox1(new BigDecimal(line[0].replace(",", ".")));
+                stationDto.setSdoy1(new BigDecimal(line[1].replace(",", ".")));
                 stationDto.setG3efid(Long.parseLong(line[2]));
                 stationDto.setStationId(line[3]);
                 stationDto.setStationName(line[4]);
@@ -271,26 +306,18 @@ public class AddressImportService {
                 stationDto.setLongitude(decimals.get(0));
                 stationDto.setLatitude(decimals.get(1));
 
-                StationDto savedStationDto;
                 if (isTableClean) {
-                    savedStationDto = stationService.insertStation(stationDto);
+                    stationService.insertStation(stationDto);
                 } else {
-                    savedStationDto = stationService.updateOrInsertStationByG3efid(stationDto);
+                    stationService.updateOrInsertStationByG3efid(stationDto);
                 }
 
-               /* System.out.println("Adresse [sdo_x1= " + line[0]
-                        + ", sdo_y1= " + line[1]
-                        + ", g3e_fid=" + line[2]
-                        + ", plz=" + line[3]
-                        + ", ort=" + line[4]
-                        + ", ortsteil=" + line[5]
-                        + ", strasse=" + line[6]
-                        + ", hausnummer=" + line[7]
-                        + "]");*/
             }
         } catch (IOException | CsvValidationException e) {
-            e.printStackTrace();
+            log.error(e);
         }
+        Instant end = Instant.now();
+        log.info("Duration: " + Duration.between(start, end));
     }
 
 }

addressImport/src/main/java/org/eclipse/openk/gridfailureinformation/importadresses/service/AddressService.java

@@ -72,7 +72,8 @@ public class AddressService {
 
         } catch (Exception ex) {
 
-            log.info("Adresse [sdo_x1= " + addressDto.getSdox1()
+            log.error(ex);
+            log.warn("Adresse [sdo_x1= " + addressDto.getSdox1()
                     + ", sdo_y1= " + addressDto.getSdoy1()
                     + ", g3e_fid=" + addressDto.getG3efid()
                     + ", plz=" + addressDto.getPostcode()
@@ -87,6 +88,6 @@ public class AddressService {
     }
 
     public  void deleteAllAddresses() {
-        addressRepository.deleteAll();
+        addressRepository.deleteAllInBatch();
     }
 }

addressImport/src/main/java/org/eclipse/openk/gridfailureinformation/importadresses/service/StationService.java

@@ -59,7 +59,7 @@ public class StationService {
 
         } catch (Exception ex) {
 
-            log.info("Adresse [sdo_x1= " + stationDto.getSdox1()
+            log.warn("Adresse [sdo_x1= " + stationDto.getSdox1()
                     + ", sdo_y1= " + stationDto.getSdoy1()
                     + ", g3e_fid=" + stationDto.getG3efid()
                     + ", station_id=" + stationDto.getStationId()
@@ -72,7 +72,7 @@ public class StationService {
     }
     @Transactional
     public  void deleteAllStations() {
-        stationRepository.deleteAll();
+        stationRepository.deleteAllInBatch();
     }
 
 }

addressImport/src/main/java/org/eclipse/openk/gridfailureinformation/importadresses/util/UtmConverter.java

@@ -5,6 +5,7 @@ import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 
 import java.math.BigDecimal;
+import java.math.RoundingMode;
 import java.util.LinkedList;
 import java.util.List;
 
@@ -38,10 +39,9 @@ public class UtmConverter {
         longitude=Math.round(lon*10000000);
         longitude = longitude/10000000;
 
-       List<BigDecimal> decimals = new LinkedList<BigDecimal>();