Unverified Commit 5bed7ffc authored by Jay Billings's avatar Jay Billings Committed by GitHub

Merge pull request #461 from dbluhm/dependency-scraper

Dependency scraper
parents 17d39956 2754d16f
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" output="target/classes" path="src/main/java">
<attributes>
<attribute name="optional" value="true"/>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="src" output="target/test-classes" path="src/test/java">
<attributes>
<attribute name="test" value="true"/>
<attribute name="optional" value="true"/>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry excluding="**" kind="src" output="target/test-classes" path="src/test/resources">
<attributes>
<attribute name="test" value="true"/>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
<classpathentry kind="output" path="target/classes"/>
</classpath>
<?xml version="1.0" encoding="UTF-8"?>
<projectDescription>
<name>org.eclipse.ice.dev.dependencyscraper</name>
<comment></comment>
<projects>
</projects>
<buildSpec>
<buildCommand>
<name>org.eclipse.jdt.core.javabuilder</name>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
<name>org.eclipse.m2e.core.maven2Builder</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.jdt.core.javanature</nature>
<nature>org.eclipse.m2e.core.maven2Nature</nature>
</natures>
</projectDescription>
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>org.eclipse.ice.dev</artifactId>
<groupId>org.eclipse.ice</groupId>
<version>3.0.0-SNAPSHOT</version>
</parent>
<artifactId>org.eclipse.ice.dev.dependencyscraper</artifactId>
<packaging>maven-plugin</packaging>
<name>org.eclipse.ice.dev.dependencyscraper Maven Plugin</name>
<!-- FIXME change it to the project's website -->
<url>http://www.example.com</url>
<prerequisites>
<maven>${maven.version}</maven>
</prerequisites>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>11</maven.compiler.source>
<maven.compiler.target>11</maven.compiler.target>
<maven.compiler.release>11</maven.compiler.release>
<maven.version>3.3.9</maven.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-plugin-api</artifactId>
<version>${maven.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-core</artifactId>
<version>${maven.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-artifact</artifactId>
<version>${maven.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-compat</artifactId>
<version>${maven.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.maven.plugin-tools</groupId>
<artifactId>maven-plugin-annotations</artifactId>
<version>3.6.0</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.maven.plugin-testing</groupId>
<artifactId>maven-plugin-testing-harness</artifactId>
<version>3.3.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.6</version>
</dependency>
</dependencies>
<build>
<pluginManagement><!-- lock down plugins versions to avoid using Maven defaults (may be moved to parent pom) -->
<plugins>
<plugin>
<artifactId>maven-clean-plugin</artifactId>
<version>3.1.0</version>
</plugin>
<!-- see http://maven.apache.org/ref/current/maven-core/default-bindings.html#Plugin_bindings_for_maven-plugin_packaging -->
<plugin>
<artifactId>maven-resources-plugin</artifactId>
<version>3.0.2</version>
</plugin>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.0</version>
</plugin>
<plugin>
<artifactId>maven-plugin-plugin</artifactId>
<version>3.6.0</version>
</plugin>
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.22.1</version>
</plugin>
<plugin>
<artifactId>maven-jar-plugin</artifactId>
<version>3.0.2</version>
</plugin>
<plugin>
<artifactId>maven-install-plugin</artifactId>
<version>2.5.2</version>
</plugin>
<plugin>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.8.2</version>
</plugin>
<plugin>
<artifactId>maven-invoker-plugin</artifactId>
<version>3.1.0</version>
</plugin>
</plugins>
</pluginManagement>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-plugin-plugin</artifactId>
<version>3.6.0</version>
<configuration>
<!-- <goalPrefix>maven-archetype-plugin</goalPrefix> -->
<skipErrorNoDescriptorsFound>true</skipErrorNoDescriptorsFound>
</configuration>
<executions>
<execution>
<id>mojo-descriptor</id>
<goals>
<goal>descriptor</goal>
</goals>
</execution>
<execution>
<id>help-goal</id>
<goals>
<goal>helpmojo</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
/*******************************************************************************
* Copyright (c) 2020- UT-Battelle, LLC.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*
* Contributors:
* Daniel Bluhm - Initial implementation
*******************************************************************************/
package org.org.eclipse.ice.dev.dependencyscraper;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
import org.apache.maven.artifact.Artifact;
import org.apache.maven.plugin.AbstractMojo;
import org.apache.maven.plugin.MojoExecutionException;
import org.apache.maven.plugin.MojoFailureException;
import org.apache.maven.plugins.annotations.LifecyclePhase;
import org.apache.maven.plugins.annotations.Mojo;
import org.apache.maven.plugins.annotations.Parameter;
import org.apache.maven.plugins.annotations.ResolutionScope;
import org.apache.maven.project.MavenProject;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.List;
import java.util.Locale;
import java.util.Set;
import java.util.jar.JarFile;
import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
/**
* Maven plugin used to search dependencies for files matching filtering rules
* and copy to output directory.
*/
@Mojo(
name = "scrape",
defaultPhase = LifecyclePhase.PROCESS_SOURCES,
requiresDependencyResolution = ResolutionScope.COMPILE_PLUS_RUNTIME
)
public class DependencyScraper extends AbstractMojo {
/**
* Prefix used for generated temp files.
*/
private static final String TEMP_PREFIX = "org.eclipse.ice.dev.dependencyscraper";
/**
* Current project. Used to get handle to dependencies.
*/
@Parameter(
defaultValue = "${project}",
readonly = true,
required = true
)
private MavenProject project;
/**
* Directory to output scraped files.
*/
@Parameter(
property = "outputDirectory",
required = true
)
private File outputDirectory;
@Parameter(
property = "sourceDirectory",
required = true
)
private String sourceDirectory;
/**
* List of files to include. May contain wildcards, i.e. <code>*</code> or
* <code>**{@literal /}*.js</code>.
*/
@Parameter(
property = "includes",
required = true
)
private List<String> includes;
/**
* Whether files should be overwritten if they already exist.
*/
@Parameter(
property = "clobber",
required = false,
defaultValue = "false"
)
private boolean clobber;
/**
* Set of jar files that will be searched for matching files.
*/
private Set<File> jarFiles;
/**
* Setter for outputDirectory.
* @param outputDirectory to set.
*/
public void setOutputDirectory(File outputDirectory) {
this.outputDirectory = outputDirectory;
}
/**
* Setter for includes.
* @param includes to set.
*/
public void setIncludes(List<String> includes) {
this.includes = includes;
}
/**
* Setter for clobber.
* @param clobber to set.
*/
public void setClobber(boolean clobber) {
this.clobber = clobber;
}
/**
* Setter for jarFiles.
* @param jarFiles to set.
*/
public void setJarFiles(Set<File> jarFiles) {
this.jarFiles = jarFiles;
}
/**
* Filter for whether a ZipEntry begins with the sourceDirectory.
* @param file to test
* @return if ZipEntry path begins with sourceDirectory.
*/
private boolean startsWithSourceDirectory(ZipEntry file) {
return file.getName()
.toLowerCase(Locale.ENGLISH)
.startsWith(sourceDirectory.toLowerCase(Locale.ENGLISH));
}
/**
* Filter for whether a ZipEntry should be included.
* @param file to test
* @return if it should be included
*/
private boolean shouldInclude(ZipEntry file) {
return includes.stream()
.anyMatch(
include -> FilenameUtils.wildcardMatch(file.getName(), include)
);
}
/**
* Efficient and safe copy, only overwriting if destination differs.
* @param is input stream source
* @param dest output file
* @throws IOException on permission errors, etc..
*/
private void copyInputToDestIfDiffers(InputStream is, File dest) throws IOException {
File tempFile = File.createTempFile(TEMP_PREFIX, null);
FileUtils.copyInputStreamToFile(is, tempFile);
if (!FileUtils.contentEquals(tempFile, dest)) {
FileUtils.forceDelete(dest);
FileUtils.moveFile(tempFile, dest);
} else {
tempFile.delete();
}
}
/**
* Copy InputStream to file, respecting clobber settings.
* @param is input stream source
* @param dest output file
* @throws IOException on permission errors, etc..
*/
private void copyRespectingClobber(InputStream is, File dest) throws IOException {
if (dest.exists()) {
if (clobber) { // Overwrite existing file
getLog().info(String.format(
"File %s already exists and clobber is set; overwriting.",
dest.getName()
));
copyInputToDestIfDiffers(is, dest);
} else {
getLog().info(String.format(
"File %s already exists and clobber is not set; skipping.",
dest.getName()
));
}
} else {
FileUtils.copyInputStreamToFile(is, dest);
}
}
/**
* Copy file from jar to output directory.
* @param jar from which file will be copied
* @param fileInJar file to copy
* @throws MojoFailureException if file could not be copied, inadequate
* permissions, etc.
*/
private void copyFileInJarToOutput(
JarFile jar, ZipEntry fileInJar
) throws MojoFailureException {
String fullPath = fileInJar.getName();
// Drop sourceDirectory from file path
String relativePath = fullPath
.substring(fullPath.toLowerCase(Locale.ENGLISH)
.indexOf(sourceDirectory.toLowerCase(Locale.ENGLISH))
+ sourceDirectory.length());
// Determine output path
File target = new File(outputDirectory, relativePath);
try {
copyRespectingClobber(jar.getInputStream(fileInJar), target);
} catch (IOException e) {
throw new MojoFailureException(
"Failed to copy file " + fileInJar.getName()
);
}
}
/**
* Collect Jar files from dependencies and search through them for files
* matching the parameters of the plugin. Copy matching files into output
* directory specified by parameters.
*/
public void execute() throws MojoExecutionException, MojoFailureException {
if (jarFiles == null) {
this.jarFiles = project.getArtifacts().stream()
// Only process jar dependencies
.filter(artifact -> "jar".equals(artifact.getType()))
// Map to File
.map(Artifact::getFile)
.collect(Collectors.toSet());
}
for (File jar : jarFiles) {
try (JarFile jarFile = new JarFile(jar, false)) {
Set<ZipEntry> toCopy = jarFile.stream()
// Filter out directories
.filter(file -> !file.isDirectory())
// Filter out any files not in the source directory
.filter(this::startsWithSourceDirectory)
// Filter out files not matching any include wildcards
.filter(this::shouldInclude)
.collect(Collectors.toSet());
for (ZipEntry file : toCopy) {
getLog().info(String.format("Copying %s", file.getName()));
copyFileInJarToOutput(jarFile, file);
}
} catch (IOException e) {
throw new MojoFailureException(
String.format("Failed to open jar file %s!", jar.toString())
);
}
}
}
}
\ No newline at end of file
package org.org.eclipse.ice.dev.dependencyscraper;
import org.apache.commons.io.FileUtils;
import org.apache.maven.plugin.testing.MojoRule;
import org.apache.maven.plugin.MojoFailureException;
import org.junit.After;
import org.junit.Rule;
import static org.junit.Assert.*;
import org.junit.Test;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.List;
import java.util.Set;
/**
* Test DepedencyScraper Maven Plugin.
* @author Daniel Bluhm
*/
public class DependencyScraperTest
{
/**
* Test "project" path.
*/
private static final Path PROJECT = Path.of("target/test-classes/project-to-test");
/**
* Test project's output path.
*/
private static final Path OUTPUT = Path.of("target/test-classes/project-to-test/test");
/**
* Maven Plugin Testing Harness magic that enables us to get a dependency
* scraper instance from reading a specified POM file.
*/
@Rule
public MojoRule rule = new MojoRule() {
@Override
protected void before() throws Throwable { }
@Override
protected void after() { }
};
/**
* Get an instance of the DependencyScraper with pretend jar dep already
* set.
* @return
* @throws Exception
*/
private DependencyScraper getMojo() throws Exception {
File pom = PROJECT.toFile();
assertNotNull(pom);
assertTrue(pom.exists());
DependencyScraper myMojo = (DependencyScraper) rule.lookupConfiguredMojo(pom, "scrape");
assertNotNull(myMojo);
myMojo.setJarFiles(Set.of(
PROJECT.resolve("pretend_dependency.jar").toFile()
));
return myMojo;
}
/**
* Clean out the output directory after each test.
* @throws IOException if any
*/
@After
public void clearOutputDirectory() throws IOException {
if (Files.exists(OUTPUT)) {
FileUtils.cleanDirectory(OUTPUT.toFile());
}
}
/**
* Test that the DependencyScraper reads the pom and copies the expected
* file.
* @throws Exception if any
*/
@Test
public void testExecuteWorks() throws Exception {
DependencyScraper myMojo = getMojo();
myMojo.execute();
assertTrue(OUTPUT.toFile().exists());
assertTrue(OUTPUT.resolve("test.txt").toFile().exists());
}
/**
* Test that the DependencyScraper reads the pom and runs but does not do
* anything when no matching jars to scrape.
* @throws Exception if any
*/
@Test
public void testExecuteWhenNoJars() throws Exception {
DependencyScraper myMojo = getMojo();
myMojo.setJarFiles(null);
myMojo.execute();