From 150193ccb47d1b85802990f7acd7e8bac314ad53 Mon Sep 17 00:00:00 2001
From: bourgesl <bourges.laurent@gmail.com>
Date: Tue, 27 Mar 2018 12:18:49 +0200
Subject: [PATCH] improved remove pipeline (more robust if some folders are
missing + delete orphan projects)
---
src/main/java/fr/osug/DOIApplication.java | 8 +-
.../java/fr/osug/doi/PipelineCommonData.java | 13 +-
src/main/java/fr/osug/doi/ProjectConfig.java | 172 ++++++++++--------
src/main/java/fr/osug/doi/RemovePipeline.java | 72 ++++++--
.../java/fr/osug/doi/service/DoiService.java | 38 +++-
.../java/fr/osug/doi/DOITextToXmlTest.java | 2 +-
6 files changed, 203 insertions(+), 102 deletions(-)
diff --git a/src/main/java/fr/osug/DOIApplication.java b/src/main/java/fr/osug/DOIApplication.java
index 2c557d3..9042489 100644
--- a/src/main/java/fr/osug/DOIApplication.java
+++ b/src/main/java/fr/osug/DOIApplication.java
@@ -208,7 +208,7 @@ public class DOIApplication {
if (Character.toLowerCase(ch) == 'y') {
// Staging only
- final AbstractPipeline<?> pipeline = doiConfig.getRemovePipeline(getProjectConfig(project), doiPattern);
+ final AbstractPipeline<?> pipeline = doiConfig.getRemovePipeline(getProjectConfig(project, false), doiPattern);
pipeline.execute();
} else {
logger.info("Action cancelled ({})", ch);
@@ -238,7 +238,11 @@ public class DOIApplication {
}
private ProjectConfig getProjectConfig(final String project) throws IOException {
- return new ProjectConfig(doiConfig.getPathConfig(), project);
+ return getProjectConfig(project, true);
+ }
+
+ private ProjectConfig getProjectConfig(final String project, final boolean required) throws IOException {
+ return new ProjectConfig(doiConfig.getPathConfig(), project, required);
}
/** Show command arguments help. */
diff --git a/src/main/java/fr/osug/doi/PipelineCommonData.java b/src/main/java/fr/osug/doi/PipelineCommonData.java
index 032ccbf..a74d437 100644
--- a/src/main/java/fr/osug/doi/PipelineCommonData.java
+++ b/src/main/java/fr/osug/doi/PipelineCommonData.java
@@ -38,7 +38,12 @@ public class PipelineCommonData {
projectConfigs.put(projectConfig.getProjectName(), projectConfig);
addProjectConfigAll(projectConfig);
}
-
+
+ public void removeProjectConfig(final ProjectConfig projectConfig) {
+ projectConfigs.remove(projectConfig.getProjectName());
+ removeProjectConfigAll(projectConfig);
+ }
+
public Collection<ProjectConfig> getProjectConfigs() {
return projectConfigs.values();
}
@@ -47,10 +52,14 @@ public class PipelineCommonData {
projectConfigsAll.put(projectConfig.getProjectName(), projectConfig);
}
+ public void removeProjectConfigAll(final ProjectConfig projectConfig) {
+ projectConfigsAll.remove(projectConfig.getProjectName());
+ }
+
public Collection<ProjectConfig> getProjectConfigsAll() {
return projectConfigsAll.values();
}
-
+
public ProjectConfig getProjectConfigAll(final String projectName) throws IOException {
ProjectConfig projectConfig = projectConfigsAll.get(projectName);
if (projectConfig == null) {
diff --git a/src/main/java/fr/osug/doi/ProjectConfig.java b/src/main/java/fr/osug/doi/ProjectConfig.java
index b15d82f..31389f0 100644
--- a/src/main/java/fr/osug/doi/ProjectConfig.java
+++ b/src/main/java/fr/osug/doi/ProjectConfig.java
@@ -71,55 +71,66 @@ public final class ProjectConfig {
private Map<String, NameEntry> overrideNameEntryMap = null;
public ProjectConfig(final PathConfig pathConfig, final String projectName) throws IOException {
+ this(pathConfig, projectName, true);
+ }
+
+ public ProjectConfig(final PathConfig pathConfig, final String projectName, final boolean required) throws IOException {
this.projectName = projectName;
// may fail:
- this.projectConf = FileUtils.getRequiredDirectory(Paths.DIR_CONFIG + projectName).getCanonicalFile();
+ this.projectConf = dir(absPath(Paths.DIR_CONFIG, projectName), required);
logger.info("Project Config: {}", projectConf);
- // preload properties to ensure the folder exists:
- getProperties();
-
- initMetadataDir(
- new File(this.projectConf, Paths.DIR_PROJECT_METADATA).getAbsolutePath()
- );
- initInputDir(
- new File(this.projectConf, Paths.DIR_PROJECT_INPUTS).getAbsolutePath()
- );
- initTmpDir(
- new File(Paths.DIR_TMP, projectName).getAbsolutePath()
- );
- initTmpDoiDir(
- new File(tmpDir, Paths.DIR_TMP_DOI).getAbsolutePath()
- );
-
- initStagingDir(
- new File(Paths.DIR_STAGING, projectName).getAbsolutePath()
- );
- initPublicDir(
- new File(Paths.DIR_PUBLIC, projectName).getAbsolutePath()
- );
+ if (required) {
+ // preload properties to ensure the folder contains the property file:
+ getProperties();
+ }
+
+ initInputDir(absPath(this.projectConf, Paths.DIR_PROJECT_INPUTS), required);
+ initMetadataDir(absPath(this.projectConf, Paths.DIR_PROJECT_METADATA), required);
+
+ initTmpDir(absPath(Paths.DIR_TMP, projectName));
+ initTmpDoiDir(absPath(tmpDir, Paths.DIR_TMP_DOI));
+
+ initStagingDir(absPath(Paths.DIR_STAGING, projectName));
+ initPublicDir(absPath(Paths.DIR_PUBLIC, projectName));
// /www/staging directories:
- initWebStagingProjectDir(
- new File(pathConfig.getWebStagingDir(), projectName).getAbsolutePath()
- );
- initWebStagingProjectEmbedDir(
- new File(webStagingProjectDir, Paths.DIR_WEB_EMBED).getAbsolutePath()
- );
- initWebStagingProjectXmlDir(
- new File(webStagingProjectDir, Paths.DIR_WEB_XML).getAbsolutePath()
- );
+ initWebStagingProjectDir(absPath(pathConfig.getWebStagingDir(), projectName));
+ initWebStagingProjectEmbedDir(absPath(webStagingProjectDir, Paths.DIR_WEB_EMBED));
+ initWebStagingProjectXmlDir(absPath(webStagingProjectDir, Paths.DIR_WEB_XML));
+
// /www/public directories:
- initWebPublicProjectDir(
- new File(pathConfig.getWebPublicDir(), projectName).getAbsolutePath()
- );
- initWebPublicProjectEmbedDir(
- new File(webPublicProjectDir, Paths.DIR_WEB_EMBED).getAbsolutePath()
- );
- initWebPublicProjectXmlDir(
- new File(webPublicProjectDir, Paths.DIR_WEB_XML).getAbsolutePath()
- );
+ initWebPublicProjectDir(absPath(pathConfig.getWebPublicDir(), projectName));
+ initWebPublicProjectEmbedDir(absPath(webPublicProjectDir, Paths.DIR_WEB_EMBED));
+ initWebPublicProjectXmlDir(absPath(webPublicProjectDir, Paths.DIR_WEB_XML));
+ }
+
+ public String dumpProjectDirs() {
+ final StringBuilder sb = new StringBuilder(1024);
+
+ sb.append(getProjectConf()).append('\n');
+ sb.append(getInputDir()).append('\n');
+ sb.append(getMetadataDir()).append('\n');
+ sb.append(getTemplatesDir()).append('\n');
+
+ sb.append(getTmpDir()).append('\n');
+ sb.append(getTmpDoiDir()).append('\n');
+
+ sb.append(getStagingDir()).append('\n');
+ sb.append(getPublicDir()).append('\n');
+
+ // /www/staging directories:
+ sb.append(getWebStagingProjectDir()).append('\n');
+ sb.append(getWebPublicProjectEmbedDir()).append('\n');
+ sb.append(getWebPublicProjectXmlDir()).append('\n');
+
+ // /www/public directories:
+ sb.append(getWebPublicProjectDir()).append('\n');
+ sb.append(getWebPublicProjectEmbedDir()).append('\n');
+ sb.append(getWebPublicProjectXmlDir()).append('\n');
+
+ return sb.toString();
}
public void prepareProcess() throws IOException {
@@ -130,62 +141,58 @@ public final class ProjectConfig {
getOverrideNameMap();
}
- public void initMetadataDir(final String dirPath) throws IOException {
- this.metadataDir = FileUtils.getRequiredDirectory(dirPath).getCanonicalFile();
+ public void initInputDir(final String dirPath, final boolean required) throws IOException {
+ this.inputDir = dir(dirPath, required);
}
- public void initInputDir(final String dirPath) throws IOException {
- this.inputDir = FileUtils.getRequiredDirectory(dirPath).getCanonicalFile();
+ public void initMetadataDir(final String dirPath, final boolean required) throws IOException {
+ this.metadataDir = dir(dirPath, required);
}
- public void initTmpDir(final String path) throws IOException {
- this.tmpDir = FileUtils.createDirectories(path);
+ public void initTmpDir(final String dirPath) throws IOException {
+ this.tmpDir = FileUtils.createDirectories(dirPath);
}
- public void initTmpDoiDir(final String path) throws IOException {
- this.tmpDoiDir = FileUtils.createDirectories(path);
+ public void initTmpDoiDir(final String dirPath) throws IOException {
+ this.tmpDoiDir = FileUtils.createDirectories(dirPath);
}
- public void initStagingDir(final String path) throws IOException {
- this.stagingDir = FileUtils.createDirectories(path);
+ public void initStagingDir(final String dirPath) throws IOException {
+ this.stagingDir = FileUtils.createDirectories(dirPath);
}
- public void initPublicDir(final String path) throws IOException {
- this.publicDir = FileUtils.createDirectories(path);
+ public void initPublicDir(final String dirPath) throws IOException {
+ this.publicDir = FileUtils.createDirectories(dirPath);
}
- public void initWebStagingProjectDir(final String path) throws IOException {
- this.webStagingProjectDir = FileUtils.createDirectories(path);
+ public void initWebStagingProjectDir(final String dirPath) throws IOException {
+ this.webStagingProjectDir = FileUtils.createDirectories(dirPath);
}
- public void initWebStagingProjectEmbedDir(final String path) throws IOException {
- this.webStagingProjectEmbedDir = FileUtils.createDirectories(path);
+ public void initWebStagingProjectEmbedDir(final String dirPath) throws IOException {
+ this.webStagingProjectEmbedDir = FileUtils.createDirectories(dirPath);
}
- public void initWebStagingProjectXmlDir(final String path) throws IOException {
- this.webStagingProjectXmlDir = FileUtils.createDirectories(path);
+ public void initWebStagingProjectXmlDir(final String dirPath) throws IOException {
+ this.webStagingProjectXmlDir = FileUtils.createDirectories(dirPath);
}
- public void initWebPublicProjectDir(final String path) throws IOException {
- this.webPublicProjectDir = FileUtils.createDirectories(path);
+ public void initWebPublicProjectDir(final String dirPath) throws IOException {
+ this.webPublicProjectDir = FileUtils.createDirectories(dirPath);
}
- public void initWebPublicProjectEmbedDir(final String path) throws IOException {
- this.webPublicProjectEmbedDir = FileUtils.createDirectories(path);
+ public void initWebPublicProjectEmbedDir(final String dirPath) throws IOException {
+ this.webPublicProjectEmbedDir = FileUtils.createDirectories(dirPath);
}
- public void initWebPublicProjectXmlDir(final String path) throws IOException {
- this.webPublicProjectXmlDir = FileUtils.createDirectories(path);
+ public void initWebPublicProjectXmlDir(final String dirPath) throws IOException {
+ this.webPublicProjectXmlDir = FileUtils.createDirectories(dirPath);
}
public String getProjectName() {
return projectName;
}
- public File getProjectConf() {
- return projectConf;
- }
-
/* properties */
private Properties getProperties() {
if (properties == null) {
@@ -244,9 +251,7 @@ public final class ProjectConfig {
/* templates */
public DoiTemplates getTemplates() throws IOException {
if (templates == null) {
- templates = new DoiTemplates(
- new File(this.projectConf, Paths.DIR_PROJECT_TEMPLATES).getAbsolutePath()
- );
+ templates = new DoiTemplates(getTemplatesDir());
}
return templates;
}
@@ -264,6 +269,14 @@ public final class ProjectConfig {
}
/* Directories */
+ public File getProjectConf() {
+ return projectConf;
+ }
+
+ private String getTemplatesDir() {
+ return absPath(this.projectConf, Paths.DIR_PROJECT_TEMPLATES);
+ }
+
public File getMetadataDir() {
return metadataDir;
}
@@ -407,4 +420,19 @@ public final class ProjectConfig {
logger.debug("nameEntryMap: {}", nameEntryMap);
return nameEntryMap;
}
+
+ private File dir(final String dirPath, final boolean required) throws IOException {
+ if (required) {
+ return FileUtils.getRequiredDirectory(dirPath).getCanonicalFile();
+ }
+ return FileUtils.createDirectories(dirPath);
+ }
+
+ private static String absPath(final String dir, final String name) {
+ return new File(dir, name).getAbsolutePath();
+ }
+
+ private static String absPath(final File dir, final String name) {
+ return new File(dir, name).getAbsolutePath();
+ }
}
diff --git a/src/main/java/fr/osug/doi/RemovePipeline.java b/src/main/java/fr/osug/doi/RemovePipeline.java
index 4a70905..dd3e372 100644
--- a/src/main/java/fr/osug/doi/RemovePipeline.java
+++ b/src/main/java/fr/osug/doi/RemovePipeline.java
@@ -6,12 +6,15 @@ package fr.osug.doi;
import static fr.osug.doi.GeneratePipeline.HTML_EXT;
import fr.osug.doi.domain.Doi;
import fr.osug.doi.domain.DoiStaging;
+import fr.osug.doi.domain.Project;
import fr.osug.doi.domain.Status;
import fr.osug.doi.repository.DoiStagingRepository;
+import fr.osug.doi.repository.ProjectRepository;
import fr.osug.doi.service.DoiService;
import fr.osug.util.FileUtils;
import java.io.File;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import org.slf4j.Logger;
@@ -32,45 +35,76 @@ public final class RemovePipeline extends AbstractPipeline<PipelineCommonData> {
@Override
public void doExecute() throws IOException {
- for (ProjectConfig projectConfig : pipeData.getProjectConfigs()) {
- remove(projectConfig, pipeData.getDoiPattern());
+ // Copy current project configs:
+ final List<ProjectConfig> projectConfigs = new ArrayList<ProjectConfig>(pipeData.getProjectConfigs());
+
+ for (ProjectConfig projectConfig : projectConfigs) {
+ if (remove(projectConfig, pipeData.getDoiPattern())) {
+ pipeData.removeProjectConfig(projectConfig);
+
+ logger.warn("Please remove project paths:\n{}", projectConfig.dumpProjectDirs());
+ }
}
}
- private void remove(final ProjectConfig projectConfig, final String doiPattern) throws IOException {
+ private boolean remove(final ProjectConfig projectConfig, final String doiPattern) throws IOException {
logger.info("remove ...");
+ boolean skip = false;
+
final String pattern = DoiService.convertPattern(doiPattern);
logger.debug("pattern: {}", pattern);
final String projectName = projectConfig.getProjectName();
- final Date now = pipeData.now;
+ logger.info("project: {}", projectName);
final DoiService doiService = doiConfig.getDoiService();
- final DoiStagingRepository dsr = doiService.getDoiStagingRepository();
+ final ProjectRepository pr = doiService.getProjectRepository();
- final List<DoiStaging> dsList = dsr.findByProjectAndPattern(projectName, pattern);
+ final Project p = pr.findOneByName(projectName);
- logger.debug("DoiCommon list for project[{}]: {}", projectName, dsList);
+ if (p == null) {
+ logger.warn("Unknown Project [{}]", projectName);
+ skip = true;
+ } else {
+ final Date now = pipeData.now;
- boolean changed = false;
+ final DoiStagingRepository dsr = doiService.getDoiStagingRepository();
- for (DoiStaging ds : dsList) {
- logger.debug("DoiStaging: {}", ds);
+ final List<DoiStaging> dsList = dsr.findByProjectAndPattern(projectName, pattern);
- final Doi doi = ds.getDoi();
- final String doiSuffix = doi.getIdentifier();
+ logger.debug("DoiCommon list for project[{}]: {}", projectName, dsList);
+
+ if (dsList.isEmpty()) {
+ logger.info("Project {} has no DOI", projectName);
+ logger.info("removing {}", projectName);
+ doiService.removeOrphanProject(projectName);
+ skip = true;
- if (doi.getStatus() == Status.PUBLIC) {
- logger.warn("Ignoring {} - DOI is public", doiSuffix);
} else {
- changed |= remove(projectConfig, ds);
+ boolean changed = false;
+
+ for (DoiStaging ds : dsList) {
+ logger.debug("DoiStaging: {}", ds);
+
+ final Doi doi = ds.getDoi();
+ final String doiSuffix = doi.getIdentifier();
+
+ if (doi.getStatus() == Status.PUBLIC) {
+ logger.warn("Ignoring {} - DOI is public", doiSuffix);
+ } else {
+ changed |= remove(projectConfig, ds);
+ }
+ }
+ if (changed) {
+ // Update Project:
+ doiService.updateProjectDate(projectName, now);
+ }
}
}
- if (changed) {
- // Update Project:
- doiService.updateProjectDate(projectName, now);
- }
+
+ logger.info("remove: done.");
+ return skip;
}
private boolean remove(final ProjectConfig projectConfig, final DoiStaging ds) throws IOException {
diff --git a/src/main/java/fr/osug/doi/service/DoiService.java b/src/main/java/fr/osug/doi/service/DoiService.java
index c53ecc4..14c17f2 100644
--- a/src/main/java/fr/osug/doi/service/DoiService.java
+++ b/src/main/java/fr/osug/doi/service/DoiService.java
@@ -14,6 +14,7 @@ import fr.osug.doi.repository.DoiRepository;
import fr.osug.doi.repository.DoiStagingRepository;
import fr.osug.doi.repository.ProjectRepository;
import java.util.Date;
+import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@@ -73,6 +74,26 @@ public class DoiService {
projectRepository.save(project);
}
+ @Transactional
+ public void removeOrphanProject(final String name) {
+ // Check no DOI references this project:
+ final List<Doi> dois = doiRepository.findByProject(name);
+
+ if (!dois.isEmpty()) {
+ throw new IllegalArgumentException("Project [" + name + "] is not empty - can not remove !");
+ }
+
+ Project project = projectRepository.findOneByName(name);
+
+ if (project == null) {
+ throw new IllegalArgumentException("Missing project with name: " + name);
+ }
+
+ logger.debug("deleting: {}", project);
+
+ projectRepository.delete(project);
+ }
+
@Transactional(readOnly = true)
public Doi getDoi(final String identifier) {
return doiRepository.findOneByIdentifier(identifier);
@@ -273,7 +294,7 @@ public class DoiService {
return changed;
}
-
+
@Transactional
public void removeStagingDoiDatacite(final String identifier) {
@@ -289,7 +310,7 @@ public class DoiService {
logger.debug("deleting: {}", doiStaging);
doiStagingRepository.delete(doiStaging);
-
+
logger.debug("deleting: {}", doiStaging.getDoi());
doiRepository.delete(doiStaging.getDoi());
}
@@ -406,12 +427,17 @@ public class DoiService {
public static String convertPattern(final String doiPattern) {
String pattern;
- // escape '%' and '_'
- pattern = doiPattern.replaceAll("_", "\\_");
- pattern = pattern.replaceAll("%", "\\%");
+
+ if ("*".equals(doiPattern)) {
+ pattern = "";
+ } else {
+ // escape '%' and '_'
+ pattern = doiPattern.replaceAll("_", "\\_");
+ pattern = pattern.replaceAll("%", "\\%");
+ }
// '<pattern>%'
return pattern + "%";
}
-
+
}
diff --git a/src/test/java/fr/osug/doi/DOITextToXmlTest.java b/src/test/java/fr/osug/doi/DOITextToXmlTest.java
index f2f36af..d7d898b 100644
--- a/src/test/java/fr/osug/doi/DOITextToXmlTest.java
+++ b/src/test/java/fr/osug/doi/DOITextToXmlTest.java
@@ -91,7 +91,7 @@ public class DOITextToXmlTest {
final ProjectConfig projectConfig = new ProjectConfig(pathConfig, project);
// redirect outputs:
- projectConfig.initInputDir(DIR_TEST);
+ projectConfig.initInputDir(DIR_TEST, true);
projectConfig.initTmpDir(DIR_OUTPUT);
projectConfig.initStagingDir(DIR_STAGING);
--
GitLab