Unverified Commit 97bb20d8 authored by Joe Osborn's avatar Joe Osborn Committed by GitHub

Merge pull request #452 from gecage952/next

Create new build system and move commands tests
parents 02d4b145 2c189307
/target/
/bin/
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" output="target/classes" path="src/main/java">
<attributes>
<attribute name="optional" value="true"/>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="src" output="target/test-classes" path="src/test/java">
<attributes>
<attribute name="optional" value="true"/>
<attribute name="maven.pomderived" value="true"/>
<attribute name="test" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.7">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="output" path="target/classes"/>
</classpath>
<?xml version="1.0" encoding="UTF-8"?>
<projectDescription>
<name>org.eclipse.ice.build</name>
<comment></comment>
<projects>
</projects>
<buildSpec>
<buildCommand>
<name>org.eclipse.jdt.core.javabuilder</name>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
<name>org.eclipse.m2e.core.maven2Builder</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.jdt.core.javanature</nature>
<nature>org.eclipse.m2e.core.maven2Nature</nature>
</natures>
</projectDescription>
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.eclipse.ice</groupId>
<artifactId>org.eclipse.ice.parent</artifactId>
<version>2.2.1-SNAPSHOT</version>
<relativePath>../org.eclipse.ice.parent/pom.xml</relativePath>
</parent>
<groupId>org.eclipse.ice</groupId>
<artifactId>org.eclipse.ice.build</artifactId>
<version>1.0-SNAPSHOT</version>
<packaging>pom</packaging>
<name>org.eclipse.ice.build</name>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<modules>
<module>../org.eclipse.ice.parent</module>
<module>../org.eclipse.ice.dev</module>
<module>../org.eclipse.ice.archetypes</module>
<module>../org.eclipse.ice.commands</module>
</modules>
</project>
### Dependencies
All dependencies are noted in the `pom` file, and all but one are within maven central. The only non-centralized dependency is the ICE package `org.eclipse.ice.tests.data`. To install it, perform the following commands (after cloning the ICE repositiory) so that the Commands package can build successfully:
```shell
$ cd org.eclipse.ice.tests
$ mvn clean install
```
### Notes about tests
The automated testing is performed with a dummy remote host, which has private credentials. Thus, if the tests are built with the package, a significant portion of the tests will fail due to the fact that the dummy remote host credentials are not distributed publicly. To solve this, one may enter any generic remote host credentials into the file `$TEST_DATA_PATH/commands/ice-remote-creds.txt` in the following order
```
username
password
hostname
```
See the README in `org.eclipse.ice.tests/org.eclipse.ice.tests.data` for information regarding the `$TEST_DATA_PATH` environment variable; [this link](https://github.com/dbluhm/ice/blob/next/org.eclipse.ice.tests/org.eclipse.ice.tests.data/README.md) takes you to the README on the `next` branch.
The automated tests will then grab the necessary credentials from this file to run. Any valid ssh connection will work. If you still find that the tests fail, ensure that the ssh connection you are using has been logged into before from your host computer such that there is a key fingerprint associated to that host in your `~/.ssh/known_hosts` file. The Commands package requires that this key exists in order for authentication to proceed, no matter what form of authentication you use. In the event that tests fail on a host that already exists in `known_hosts` (e.g. with the error message `server key did not validate`, try deleting your `known_hosts` file (or the entries in your `known_hosts` that correspond to the host you are trying to run the tests on), logging in again to re-establish a fingerprint, and running the tests again.
Alternatively, you can set `StrictHostKeyChecking` to false in the `ConnectionManager`, which is in general not advised as it is inherently unsecure. To do this for the static `ConnectionManager`, just write:
```java
ConnectionManagerFactory.getConnectionManager().setRequireStrictHostKeyChecking(false);
```
Note that this is also a way through which ssh validation can be performed in the package for running actual remote commands/file transfers.
#### EmailHandler test
To test the `EmailUpdateHandler` class, a similar file to the ssh credential file must be created. Instead, a file in the location `$TEST_DATA_PATH/commands/ice-email-creds.txt` must exist which contains the following information:
```
email@address
password
SmtpHost
```
The EmailHandler will send an email from your own address to the same address with updates on when the job finishes. In order for this to happen, the email address must be authenticated. In the case of the tests, and for CI purposes, these authentications are placed in the above text file. For developer use, one could simply enter this information as it is entered in EmailHandlerTest, or you could implement another method (e.g. through use of the text file).
#### KeyGen Tests and Connections
Connections may be established via a public/private key pair that is generated between the local and remote host. Commands can function with ECDSA or RSA type keys. To generate an RSA key, for example, use:
```bash
$ ssh-keygen -t rsa -m PEM
$ ssh-copy-id -i ~/.ssh/keyname.pub username@hostname
```
Then you should be able to remotely login via `ssh -i /path/to/key username@hostname` without a password requirement.
For the keygen connection tests to pass, you should also create a key to a remote host that the tests expect to find. This can be done with any arbitrary remote server that you have credential access to; however, the key must be named dummyhostkey and must exist in your home `.ssh` directory. In other words, the key must be here:
```
$HOME/.ssh/dummyhostkey
```
where `$HOME` is the result returned from `System.getProperty("user.home")`.
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>org.eclipse.ice.tests.integration</artifactId>
<groupId>org.eclipse.ice</groupId>
<version>3.0.0-SNAPSHOT</version>
</parent>
<groupId>org.eclipse.ice</groupId>
<artifactId>org.eclipse.ice.tests.integration.commands</artifactId>
<version>3.0-SNAPSHOT</version>
<name>org.eclipse.ice.tests.integration.commands</name>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>11</maven.compiler.source>
<maven.compiler.target>11</maven.compiler.target>
</properties>
<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.11</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.eclipse.ice</groupId>
<artifactId>org.eclipse.ice.commands</artifactId>
<version>0.0.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
<version>1.7.30</version>
</dependency>
<dependency>
<groupId>org.eclipse.ice</groupId>
<artifactId>org.eclipse.ice.tests.data</artifactId>
<version>3.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.sshd</groupId>
<artifactId>sshd-core</artifactId>
<version>2.3.0</version>
</dependency>
<dependency>
<groupId>org.apache.sshd</groupId>
<artifactId>sshd-sftp</artifactId>
<version>2.3.0</version>
</dependency>
<dependency>
<groupId>com.sun.mail</groupId>
<artifactId>javax.mail</artifactId>
<version>1.6.2</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.10</version>
</dependency>
<dependency>
<groupId>org.eclipse.ice</groupId>
<artifactId>org.eclipse.ice.data</artifactId>
<version>3.0.0-SNAPSHOT</version>
</dependency>
</dependencies>
<build>
<pluginManagement><!-- lock down plugins versions to avoid using Maven defaults (may be moved to parent pom) -->
<plugins>
<!-- clean lifecycle, see https://maven.apache.org/ref/current/maven-core/lifecycles.html#clean_Lifecycle -->
<plugin>
<artifactId>maven-clean-plugin</artifactId>
<version>3.1.0</version>
</plugin>
<!-- default lifecycle, jar packaging: see https://maven.apache.org/ref/current/maven-core/default-bindings.html#Plugin_bindings_for_jar_packaging -->
<plugin>
<artifactId>maven-resources-plugin</artifactId>
<version>3.0.2</version>
</plugin>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.0</version>
</plugin>
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.22.1</version>
</plugin>
<plugin>
<artifactId>maven-jar-plugin</artifactId>
<version>3.0.2</version>
</plugin>
<plugin>
<artifactId>maven-install-plugin</artifactId>
<version>2.5.2</version>
</plugin>
<plugin>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.8.2</version>
</plugin>
<!-- site lifecycle, see https://maven.apache.org/ref/current/maven-core/lifecycles.html#site_Lifecycle -->
<plugin>
<artifactId>maven-site-plugin</artifactId>
<version>3.7.1</version>
</plugin>
<plugin>
<artifactId>maven-project-info-reports-plugin</artifactId>
<version>3.0.0</version>
</plugin>
</plugins>
</pluginManagement>
</build>
</project>
......@@ -9,7 +9,7 @@
* Initial API and implementation and/or initial documentation - Jay Jay Billings,
* Joe Osborn
*******************************************************************************/
package org.eclipse.ice.commands;
package org.eclipse.ice.tests.commands;
import java.io.File;
import java.io.IOException;
......
......@@ -9,7 +9,7 @@
* Initial API and implementation and/or initial documentation -
* Jay Jay Billings, Joe Osborn
*******************************************************************************/
package org.eclipse.ice.commands;
package org.eclipse.ice.tests.commands;
import java.io.File;
import java.io.IOException;
......
/*******************************************************************************
* Copyright (c) 2019- UT-Battelle, LLC.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*
* Contributors:
* Initial API and implementation and/or initial documentation -
* Jay Jay Billings, Joe Osborn
*******************************************************************************/
package org.eclipse.ice.tests.commands;
import static org.junit.Assert.*;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import org.apache.sshd.client.subsystem.sftp.SftpClient;
import org.apache.sshd.client.subsystem.sftp.SftpClient.OpenMode;
import org.apache.sshd.client.subsystem.sftp.SftpClientFactory;
import org.eclipse.ice.commands.Command;
import org.eclipse.ice.commands.CommandConfiguration;
import org.eclipse.ice.commands.CommandFactory;
import org.eclipse.ice.commands.CommandStatus;
import org.eclipse.ice.commands.Connection;
import org.eclipse.ice.commands.ConnectionAuthorizationHandler;
import org.eclipse.ice.commands.ConnectionConfiguration;
import org.eclipse.ice.commands.ConnectionManagerFactory;
import org.eclipse.ice.commands.KeyPathConnectionAuthorizationHandler;
import org.eclipse.ice.commands.RemoteRemoteFileTransferCommand;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
/**
* This class tests remote to remote file transfers executed from a local host
* A, i.e. from host A a remote file transfer between host B and host C
*
* @author Joe Osborn
*
*/
public class RemoteRemoteFileTransferTest {
/**
* A source file to use for testing
*/
private String source = "";
/**
* A destination to move the file
*/
private String destination = "/tmp/";
/**
* Connection information for host B
*/
private ConnectionConfiguration hostBConnection;
/**
* Remote host C key path that is needed to establish connection between host B
* and host C
*/
private String remoteHostCKeyPath = "/some/key/connecting/B/to/C";
/**
* Remote host B key path that is needed to connect host A to host B
*/
private String hostBKeyPath = System.getProperty("user.home") + "/.ssh/somekey";
/**
* Authorization for remote host C
*/
private KeyPathConnectionAuthorizationHandler remoteHostC;
/**
* @throws java.lang.Exception
*/
@BeforeClass
public static void setUpBeforeClass() throws Exception {
}
/**
* @throws java.lang.Exception
*/
@AfterClass
public static void tearDownAfterClass() throws Exception {
// Delete the extra output files that were made during clean up
String rm = "lsOut.txt lsErr.txt";
List<String> command = new ArrayList<String>();
// Build a command
if (System.getProperty("os.name").toLowerCase().contains("win")) {
command.add("powershell.exe");
} else {
command.add("/bin/bash");
command.add("-c");
}
command.add("rm " + rm);
// Execute the command with the process builder api
ProcessBuilder builder = new ProcessBuilder(command);
// Files exist in the top most directory of the package
String topDir = System.getProperty("user.dir");
File file = new File(topDir);
builder.directory(file);
// Process it
Process job = builder.start();
job.waitFor(); // wait for it to finish
// Remove connections
ConnectionManagerFactory.getConnectionManager().removeAllConnections();
}
/**
* A test function to test the remote to remote file transfer functionality,
* where the file transfer goes from one remote host B to another remote host C
*
* @throws Exception
*/
@Test
@Ignore // ignore for now until we get second dummy host running
public void testRemoteRemoteFileTransfer() throws Exception {
setupConnectionConfigs();
createRemoteHostBSourceFile();
// Create the command and initialize to be able to run
RemoteRemoteFileTransferCommand command = new RemoteRemoteFileTransferCommand();
command.setConnectionConfiguration(hostBConnection);
command.setRemoteHostCAuthorization(remoteHostC);
command.setConfiguration(source, destination);
command.createCommand();
CommandStatus status = command.execute();
// Check that command completed correctly
assertEquals(CommandStatus.SUCCESS, status);
// Check if file transfer was successful and deletes the moved file on host C
assertTrue(checkPathExistsRemoteHostC());
// Now clean up the file created
deleteHostBSource();
}
/**
* Deletes the source file from the remote host B
*
* @throws IOException
*/
protected void deleteHostBSource() throws IOException {
Connection conn = ConnectionManagerFactory.getConnectionManager().getConnection(hostBConnection.getName());
conn.setSftpChannel(SftpClientFactory.instance().createSftpClient(conn.getSession()));
SftpClient channel = conn.getSftpChannel();
channel.remove(source);
}
/**
* Creates a source file to play with on remote host B. Constructs a file
* locally and then moves it to remote host B
*
* @throws IOException
*/
protected void createRemoteHostBSourceFile() throws IOException {
Connection conn = ConnectionManagerFactory.getConnectionManager().openConnection(hostBConnection);
conn.setSftpChannel(SftpClientFactory.instance().createSftpClient(conn.getSession()));
SftpClient sftpChannel = conn.getSftpChannel();
String hostBsource = "/tmp/";
try {
sftpChannel.lstat(hostBsource);
} catch (Exception e) {
System.out.println("Remote directory not found, trying to make it for source file.");
sftpChannel.mkdir(hostBsource);
}
// Create a dummy file to test
String src = "remoteRemoteDummyfile.txt";
Path sourcePath = null;
try {
sourcePath = Files.createTempFile(null, src);
} catch (IOException e) {
e.printStackTrace();
}
// Get the filename by splitting the path by "/"
String separator = FileSystems.getDefault().getSeparator();
if (System.getProperty("os.name").toLowerCase().contains("win"))
separator += "\\";
String[] tokens = sourcePath.toString().split(separator);
// Get the last index of tokens, which will be the filename
String filename = tokens[tokens.length - 1];
// If it is just a directory, add the file name
if (hostBsource.endsWith("/")) {
hostBsource += filename;
}
// Actually move the file
try (OutputStream dstStream = sftpChannel.write(hostBsource, OpenMode.Create, OpenMode.Write,
OpenMode.Truncate)) {
try (InputStream srcStream = new FileInputStream(sourcePath.toString())) {
byte[] buf = new byte[32 * 1024];
while (srcStream.read(buf) > 0) {
dstStream.write(buf);
}
}
}
// Delete the local directory that was created since it is no longer needed
Path path = Paths.get(sourcePath.toString());
try {
Files.deleteIfExists(path);
} catch (NoSuchFileException e) {
// If somehow the directory doesn't exist, indicate so
System.err.format("%s: somehow this path doesn't exist... no such" + " file or directory%n", path);
e.printStackTrace();
}
// set the source string in this class to that just created
source = hostBsource;
}
/**
* Function that just sets up the connection information for the test to run.