Commit 8e95fe15 authored by Dimitrios Chalepakis's avatar Dimitrios Chalepakis
Browse files

Merge branch 'DEVELOP' of...

Merge branch 'DEVELOP' of ssh://git.eclipse.org:29418/openk-usermodules/org.eclipse.openk-usermodules.gridFailureInformation.backend into SI-381-Auswahl-aus-Adressbestand
parents 153a4841 c2e9cabc
FROM openjdk:8-jre-alpine
COPY /target/grid-failure-information.addressimport.service.jar /usr/src/cbd/
WORKDIR usr/src/cbd
CMD ["java", "-jar", "-Dspring.profiles.active=devserver", "grid-failure-information.addressimport.service.jar"]
\ No newline at end of file
config.stopBubbling = true
lombok.addLombokGeneratedAnnotation = true
\ No newline at end of file
......@@ -9,10 +9,10 @@
<relativePath/> <!-- lookup parent from repository -->
</parent>
<groupId>org.eclipse.openk</groupId>
<artifactId>grid-failure-information.backend.service</artifactId>
<artifactId>grid-failure-information.addressimport.service</artifactId>
<version>0.0.1-SNAPSHOT</version>
<name>grid-failure-information</name>
<description>Grid-Failure-Information Project for openKONSEQUENZ</description>
<description>Address Import of Grid-Failure-Information Project for openKONSEQUENZ</description>
<properties>
<maven.test.skip>false</maven.test.skip>
......@@ -26,6 +26,15 @@
<jruby-complete-version>9.0.0.0</jruby-complete-version>
<mapstruct.version>1.2.0.Final</mapstruct.version>
<flyway-core.version>6.0.8</flyway-core.version>
<flyway.sqlMigrationSeparator>__</flyway.sqlMigrationSeparator>
<flyway.url>jdbc:postgresql://entopticadirx:5432/GridFailureInfoDevServer</flyway.url>
<flyway.user>gfi_service</flyway.user>
<flyway.password>gfi_service</flyway.password>
<flyway.schemas>public</flyway.schemas>
<flyway.table>address_flyway_schema_history</flyway.table>
<flyway.locations>classpath:db/migration/</flyway.locations>
<postgresql.version>42.2.8</postgresql.version>
<lombock.version>1.18.10</lombock.version>
<h2.version>1.4.200</h2.version>
......
......@@ -9,7 +9,9 @@ import springfox.documentation.swagger2.annotations.EnableSwagger2;
public class AddressImportApplication {
public static void main(String[] args) {
SpringApplication.run(AddressImportApplication.class, args);
// passing through args is a security issue at sonar. As long as we don't know what for, we won't passthrough
// any arg
SpringApplication.run(AddressImportApplication.class, new String[0]);
}
}
......@@ -4,11 +4,10 @@ import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiResponse;
import io.swagger.annotations.ApiResponses;
import lombok.extern.log4j.Log4j2;
import org.eclipse.openk.gridfailureinformation.importadresses.service.AddressImportService;
import org.eclipse.openk.gridfailureinformation.importadresses.jobs.JobManager;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
......@@ -18,17 +17,16 @@ import org.springframework.web.bind.annotation.RestController;
public class AdressImportController {
@Autowired
private AddressImportService addressImportService;
private JobManager jobManager;
@PostMapping
@ApiOperation(value = "Import von Adressen und Stromstationen")
@ApiResponses(value = {
@ApiResponse(code = 201, message = "Adressen erfolgreich importiert"),
@ApiResponse(code = 500, message = "Konnte nicht durchgeführt werden")
})
@RequestMapping("/import")
public void startImportAddresses(
@RequestParam( value="cleanup", required = false, defaultValue = "true") boolean cleanup ) {
addressImportService.importAddresses(cleanup);
@PostMapping("/import")
public void startImportAddresses() {
jobManager.triggerStartImport();
}
}
\ No newline at end of file
/*
*******************************************************************************
* Copyright (c) 2019 Contributors to the Eclipse Foundation
*
* See the NOTICE file(s) distributed with this work for additional
* information regarding copyright ownership.
*
* This program and the accompanying materials are made available under the
* terms of the Eclipse Public License v. 2.0 which is available at
* http://www.eclipse.org/legal/epl-2.0.
*
* SPDX-License-Identifier: EPL-2.0
*******************************************************************************
*/
package org.eclipse.openk.gridfailureinformation.importadresses.exceptions;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.ResponseStatus;
@ResponseStatus(code = HttpStatus.INTERNAL_SERVER_ERROR)
public class InternalServerErrorException extends RuntimeException {
public InternalServerErrorException(String message) {
super(message);
}
}
......@@ -4,10 +4,16 @@ import lombok.extern.log4j.Log4j2;
import org.eclipse.openk.gridfailureinformation.importadresses.service.AddressImportService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpStatus;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import javax.xml.ws.http.HTTPException;
import static java.lang.Boolean.FALSE;
import static java.lang.Boolean.TRUE;
@EnableScheduling
@Component
@Log4j2
......@@ -18,18 +24,45 @@ public class JobManager {
@Autowired
AddressImportService addressImportService;
private final Object lock = new Object();
private Boolean startImport = FALSE;
// job for impoerting address and station data
// @Scheduled(initialDelay = 1000, fixedRate = 900000000)
// public void importAddressdata() {
// boolean cleanUp = true;
// addressImportService.importAddresses(true);
// }
//
// }
public void triggerStartImport() {
synchronized (lock) {
startImport = TRUE;
}
}
@Scheduled(cron="0/3 0/1 * 1/1 * *") // every 3 seconds
private void importOnTrigger() {
boolean go = false;
synchronized (lock) {
if( startImport.equals(TRUE) ) {
go = true;
startImport = FALSE;
}
}
if( go ) {
importAddressdata();
}
}
@Scheduled(cron= "${adressimport.cron}")
public void importAddressdata() {
private void importAddressdata() {
try {
addressImportService.importAddresses(cleanUp);
}
catch( HTTPException e ) {
if( e.getStatusCode() != HttpStatus.PROCESSING.value() ) {
throw( e );
}
else {
log.info("Jobmanager triggered import, but import already in work");
}
}
}
}
......@@ -11,11 +11,15 @@ import org.eclipse.openk.gridfailureinformation.importadresses.viewmodel.Address
import org.eclipse.openk.gridfailureinformation.importadresses.viewmodel.StationDto;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpStatus;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import javax.xml.ws.http.HTTPException;
import java.io.FileReader;
import java.io.IOException;
import java.math.BigDecimal;
import java.time.Duration;
import java.time.Instant;
import java.util.List;
@Service
......@@ -42,32 +46,68 @@ public class AddressImportService {
@Value("${adressimport.file.power-stations}")
public String filePowerStations;
public void importAddresses(Boolean cleanUp) {
private final Object lock = new Object();
private Boolean importInWork=Boolean.FALSE;
@Async
public void importAddresses(boolean cleanUp) {
// Don't import twice
synchronized (lock) {
if(importInWork.equals(Boolean.TRUE)) {
throw new HTTPException(HttpStatus.PROCESSING.value());
}
else {
importInWork = Boolean.TRUE;
}
}
if (cleanUp) {
Instant cleanUpStart = Instant.now();
log.info("CleanUp START");
addressService.deleteAllAddresses();
stationService.deleteAllStations();
log.info("CleanUp ENDE");
Instant cleanUpEnd = Instant.now();
log.info("CleanUp Duration: " + Duration.between(cleanUpStart, cleanUpEnd));
}
log.info("Adressimport START");
Instant importStart = Instant.now();
log.info("Import START");
log.info("- Import Addresses START");
importAddressFile(cleanUp);
log.info("Import Power Connections START");
log.info("- Import Addresses END");
log.info("- Import Power Connections START");
importPowerConnectionsAddressFile();
log.info("Import Water Connections START");
log.info("- Import Power Connections END");
log.info("- Import Water Connections START");
importWaterConnectionsAddressFile();
log.info("Import Gas Connections START");
log.info("- Import Water Connections END");
log.info("- Import Gas Connections START");
importGasConnectionsAddressFile();
log.info("Adressimport ENDE");
log.info("Import Power Stations START");
log.info("- Import Gas Connections END");
log.info("- Import PowerStations START");
importPowerStationFile(cleanUp);
log.info("Stationsimport ENDE");
log.info("- Import PowerStations END");
log.info("Import END");
Instant importEnd = Instant.now();
log.info("Total Import Duration: " + Duration.between(importStart, importEnd));
synchronized (lock) {
importInWork = Boolean.FALSE;
}
}
public void importAddressFile(boolean isTableClean) {
Instant start = Instant.now();
String csvFile = fileAddresses;
CSVReader reader = null;
......@@ -81,49 +121,42 @@ public class AddressImportService {
.build();
String[]line;
while ((line = reader.readNext()) != null) {
AddressDto addressDto = new AddressDto();
// !!! Parsing BigDecimal anpassen, locale
BigDecimal x1 = new BigDecimal(line[0].replaceAll(",", "."));
addressDto.setSdox1(new BigDecimal(line[0].replaceAll(",", ".")));
addressDto.setSdoy1(new BigDecimal(line[1].replaceAll(",", ".")));
addressDto.setG3efid(Long.parseLong(line[2]));
addressDto.setPostcode(line[3]);
addressDto.setCommunity(line[4]);
addressDto.setDistrict(line[5]);
addressDto.setStreet(line[6]);
addressDto.setHousenumber(line[7]);
List<BigDecimal> decimals = converter.convertUTMToDeg(addressDto.getSdox1().doubleValue(), addressDto.getSdoy1().doubleValue());
addressDto.setLongitude(decimals.get(0));
addressDto.setLatitude(decimals.get(1));
storeAddress(isTableClean, line);
AddressDto savedAddressDto;
if (isTableClean) {
savedAddressDto = addressService.insertAddress(addressDto);
} else {
savedAddressDto = addressService.updateOrInsertAddressByG3efid(addressDto);
}
/* System.out.println("Adresse [sdo_x1= " + line[0]
+ ", sdo_y1= " + line[1]
+ ", g3e_fid=" + line[2]
+ ", long= " + addressDto.getLongitude()
+ ", lat= " + addressDto.getLatitude()
+ ", plz=" + line[3]
+ ", ort=" + line[4]
+ ", ortsteil=" + line[5]
+ ", strasse=" + line[6]
+ ", hausnummer=" + line[7]
+ "]");*/
}
} catch (IOException | CsvValidationException e) {
e.printStackTrace();
log.error(e);
}
Instant end = Instant.now();
log.info("Duration: " + Duration.between(start, end));
}
private void storeAddress(boolean isTableClean, String[] line) {
AddressDto addressDto = new AddressDto();
// !!! Parsing BigDecimal anpassen, locale
addressDto.setSdox1(new BigDecimal(line[0].replace(",", ".")));
addressDto.setSdoy1(new BigDecimal(line[1].replace(",", ".")));
addressDto.setG3efid(Long.parseLong(line[2]));
addressDto.setPostcode(line[3]);
addressDto.setCommunity(line[4]);
addressDto.setDistrict(line[5]);
addressDto.setStreet(line[6]);
addressDto.setHousenumber(line[7]);
List<BigDecimal> decimals = converter.convertUTMToDeg(addressDto.getSdox1().doubleValue(), addressDto.getSdoy1().doubleValue());
addressDto.setLongitude(decimals.get(0));
addressDto.setLatitude(decimals.get(1));
if (isTableClean) {
addressService.insertAddress(addressDto);
} else {
addressService.updateOrInsertAddressByG3efid(addressDto);
}
}
public void importPowerConnectionsAddressFile() {
public void importPowerConnectionsAddressFile() {
Instant start = Instant.now();
String csvFile = filePowerConnections;
CSVReader reader = null;
......@@ -139,9 +172,8 @@ public class AddressImportService {
while ((line = reader.readNext()) != null) {
AddressDto addressDto = new AddressDto();
// !!! Parsing BigDecimal anpassen, locale???
BigDecimal x1 = new BigDecimal(line[0].replaceAll(",", "."));
addressDto.setSdox1(new BigDecimal(line[0].replaceAll(",", ".")));
addressDto.setSdoy1(new BigDecimal(line[1].replaceAll(",", ".")));
addressDto.setSdox1(new BigDecimal(line[0].replace(",", ".")));
addressDto.setSdoy1(new BigDecimal(line[1].replace(",", ".")));
addressDto.setG3efid(Long.parseLong(line[2]));
addressDto.setPostcode(line[3]);
addressDto.setCommunity(line[4]);
......@@ -155,16 +187,18 @@ public class AddressImportService {
addressDto.setLongitude(decimals.get(0));
addressDto.setLatitude(decimals.get(1));
AddressDto savedAddressDto = addressService.updateOrInsertAddressByG3efid(addressDto);
addressService.updateOrInsertAddressByG3efid(addressDto);
}
} catch (IOException | CsvValidationException e) {
e.printStackTrace();
log.error(e);
}
Instant end = Instant.now();
log.info("Duration: " + Duration.between(start, end));
}
public void importWaterConnectionsAddressFile() {
Instant start = Instant.now();
String csvFile = fileWaterConnections;
CSVReader reader = null;
......@@ -180,9 +214,8 @@ public class AddressImportService {
while ((line = reader.readNext()) != null) {
AddressDto addressDto = new AddressDto();
// !!! Parsing BigDecimal anpassen, locale???
BigDecimal x1 = new BigDecimal(line[0].replaceAll(",", "."));
addressDto.setSdox1(new BigDecimal(line[0].replaceAll(",", ".")));
addressDto.setSdoy1(new BigDecimal(line[1].replaceAll(",", ".")));
addressDto.setSdox1(new BigDecimal(line[0].replace(",", ".")));
addressDto.setSdoy1(new BigDecimal(line[1].replace(",", ".")));
addressDto.setG3efid(Long.parseLong(line[2]));
addressDto.setPostcode(line[3]);
addressDto.setCommunity(line[4]);
......@@ -196,15 +229,17 @@ public class AddressImportService {
addressDto.setLongitude(decimals.get(0));
addressDto.setLatitude(decimals.get(1));
AddressDto savedAddressDto = addressService.updateOrInsertAddressByG3efid(addressDto);
addressService.updateOrInsertAddressByG3efid(addressDto);
}
} catch (IOException | CsvValidationException e) {
e.printStackTrace();
log.error(e);
}
Instant end = Instant.now();
log.info("Duration: " + Duration.between(start, end));
}
public void importGasConnectionsAddressFile() {
Instant start = Instant.now();
String csvFile = fileGasConnections;
CSVReader reader = null;
......@@ -220,9 +255,8 @@ public class AddressImportService {
while ((line = reader.readNext()) != null) {
AddressDto addressDto = new AddressDto();
// !!! Parsing BigDecimal anpassen, locale???
BigDecimal x1 = new BigDecimal(line[0].replaceAll(",", "."));
addressDto.setSdox1(new BigDecimal(line[0].replaceAll(",", ".")));
addressDto.setSdoy1(new BigDecimal(line[1].replaceAll(",", ".")));
addressDto.setSdox1(new BigDecimal(line[0].replace(",", ".")));
addressDto.setSdoy1(new BigDecimal(line[1].replace(",", ".")));
addressDto.setG3efid(Long.parseLong(line[2]));
addressDto.setPostcode(line[3]);
addressDto.setCommunity(line[4]);
......@@ -236,15 +270,17 @@ public class AddressImportService {
addressDto.setLongitude(decimals.get(0));
addressDto.setLatitude(decimals.get(1));
AddressDto savedAddressDto = addressService.updateOrInsertAddressByG3efid(addressDto);
addressService.updateOrInsertAddressByG3efid(addressDto);
}
} catch (IOException | CsvValidationException e) {
e.printStackTrace();
log.error(e);
}
Instant end = Instant.now();
log.info("Duration: " + Duration.between(start, end));
}
public void importPowerStationFile(boolean isTableClean) {
Instant start = Instant.now();
String csvFile = filePowerStations;
CSVReader reader = null;
......@@ -260,9 +296,8 @@ public class AddressImportService {
while ((line = reader.readNext()) != null) {
StationDto stationDto = new StationDto();
// !!! Parsing BigDecimal anpassen, locale
BigDecimal x1 = new BigDecimal(line[0].replaceAll(",", "."));
stationDto.setSdox1(new BigDecimal(line[0].replaceAll(",", ".")));
stationDto.setSdoy1(new BigDecimal(line[1].replaceAll(",", ".")));
stationDto.setSdox1(new BigDecimal(line[0].replace(",", ".")));
stationDto.setSdoy1(new BigDecimal(line[1].replace(",", ".")));
stationDto.setG3efid(Long.parseLong(line[2]));
stationDto.setStationId(line[3]);
stationDto.setStationName(line[4]);
......@@ -271,26 +306,18 @@ public class AddressImportService {
stationDto.setLongitude(decimals.get(0));
stationDto.setLatitude(decimals.get(1));
StationDto savedStationDto;
if (isTableClean) {
savedStationDto = stationService.insertStation(stationDto);
stationService.insertStation(stationDto);
} else {
savedStationDto = stationService.updateOrInsertStationByG3efid(stationDto);
stationService.updateOrInsertStationByG3efid(stationDto);
}
/* System.out.println("Adresse [sdo_x1= " + line[0]
+ ", sdo_y1= " + line[1]
+ ", g3e_fid=" + line[2]
+ ", plz=" + line[3]
+ ", ort=" + line[4]
+ ", ortsteil=" + line[5]
+ ", strasse=" + line[6]
+ ", hausnummer=" + line[7]
+ "]");*/
}
} catch (IOException | CsvValidationException e) {
e.printStackTrace();
log.error(e);
}
Instant end = Instant.now();
log.info("Duration: " + Duration.between(start, end));
}
}
......
......@@ -72,7 +72,8 @@ public class AddressService {
} catch (Exception ex) {
log.info("Adresse [sdo_x1= " + addressDto.getSdox1()
log.error(ex);
log.warn("Adresse [sdo_x1= " + addressDto.getSdox1()
+ ", sdo_y1= " + addressDto.getSdoy1()
+ ", g3e_fid=" + addressDto.getG3efid()
+ ", plz=" + addressDto.getPostcode()
......@@ -87,6 +88,6 @@ public class AddressService {
}
public void deleteAllAddresses() {
addressRepository.deleteAll();
addressRepository.deleteAllInBatch();
}
}
......@@ -59,7 +59,7 @@ public class StationService {
} catch (Exception ex) {
log.info("Adresse [sdo_x1= " + stationDto.getSdox1()
log.warn("Adresse [sdo_x1= " + stationDto.getSdox1()
+ ", sdo_y1= " + stationDto.getSdoy1()
+ ", g3e_fid=" + stationDto.getG3efid()
+ ", station_id=" + stationDto.getStationId()
......@@ -72,7 +72,7 @@ public class StationService {
}
@Transactional
public void deleteAllStations() {
stationRepository.deleteAll();
stationRepository.deleteAllInBatch();
}
}
......@@ -5,6 +5,7 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.util.LinkedList;
import java.util.List;
......@@ -38,10 +39,9 @@ public class UtmConverter {
longitude=Math.round(lon*10000000);
longitude = longitude/10000000;
List<BigDecimal> decimals = new LinkedList<BigDecimal>();