From 9c02802042aa01b8d7154cd223656e892c3009bc Mon Sep 17 00:00:00 2001
From: bourgesl <bourges.laurent@gmail.com>
Date: Mon, 20 Feb 2017 13:54:41 +0100
Subject: [PATCH] major update: generate redirections + datacite client +
entity refactoring
---
doc/application.yml | 1 +
src/main/java/fr/osug/DOIApplication.java | 61 +-
src/main/java/fr/osug/doi/Const.java | 1 -
src/main/java/fr/osug/doi/CsvData.java | 1 -
src/main/java/fr/osug/doi/CsvUtil.java | 73 +-
src/main/java/fr/osug/doi/DOIConfig.java | 45 +-
src/main/java/fr/osug/doi/DataciteConfig.java | 60 ++
src/main/java/fr/osug/doi/DoiCsvData.java | 1 -
src/main/java/fr/osug/doi/DoiCsvSplitter.java | 1 -
src/main/java/fr/osug/doi/DoiCsvToXml.java | 18 -
src/main/java/fr/osug/doi/DoiTemplates.java | 1 -
.../java/fr/osug/doi/GeneratePipeline.java | 828 +++++++-----------
.../fr/osug/doi/GenerateRedirectPipeline.java | 208 +++++
src/main/java/fr/osug/doi/IndexUtil.java | 3 +-
src/main/java/fr/osug/doi/PathConfig.java | 281 +-----
src/main/java/fr/osug/doi/Paths.java | 21 +-
.../java/fr/osug/doi/PipelineCommonData.java | 20 +
src/main/java/fr/osug/doi/PipelineConfig.java | 12 +
.../java/fr/osug/doi/ProcessPipeline.java | 453 +++-------
.../java/fr/osug/doi/ProcessPipelineData.java | 35 +-
.../fr/osug/doi/ProcessPipelineDoiData.java | 37 +-
src/main/java/fr/osug/doi/ProjectConfig.java | 301 +++++--
src/main/java/fr/osug/doi/domain/Doi.java | 45 +-
.../java/fr/osug/doi/domain/DoiCommon.java | 139 +++
.../java/fr/osug/doi/domain/DoiPublic.java | 112 +--
.../java/fr/osug/doi/domain/DoiStaging.java | 113 +--
.../java/fr/osug/doi/domain/IndexEntry.java | 3 +-
.../java/fr/osug/doi/domain/NameEntry.java | 44 +-
src/main/java/fr/osug/doi/domain/Project.java | 29 +-
.../java/fr/osug/doi/domain/model/dbmodel.jpa | 38 +-
.../doi/repository/DoiBaseRepository.java | 25 +-
.../doi/repository/DoiPublicRepository.java | 16 +-
.../doi/repository/DoiStagingRepository.java | 14 +-
.../doi/repository/ProjectRepository.java | 5 +
.../fr/osug/doi/service/DataciteClient.java | 242 +++++
.../java/fr/osug/doi/service/DoiService.java | 77 +-
.../osug/doi/validation/ValidationUtil.java | 93 +-
.../osug/doi/validation/WarningMessage.java | 1 -
src/main/java/fr/osug/util/FileUtils.java | 41 +-
src/main/java/fr/osug/xml/XmlFactory.java | 6 +-
.../resources/config/application-default.yml | 14 +-
.../resources/config/application-test.yml | 13 +-
src/main/resources/config/application.yml | 14 +-
src/main/resources/db/migration/V1__init.sql | 22 +-
src/main/resources/logback-spring.xml | 5 +-
src/test/java/fr/osug/doi/DOIServiceTest.java | 30 +-
.../java/fr/osug/doi/DOITextToXmlTest.java | 22 +-
47 files changed, 1925 insertions(+), 1700 deletions(-)
create mode 100644 src/main/java/fr/osug/doi/DataciteConfig.java
create mode 100644 src/main/java/fr/osug/doi/GenerateRedirectPipeline.java
create mode 100644 src/main/java/fr/osug/doi/PipelineCommonData.java
create mode 100644 src/main/java/fr/osug/doi/PipelineConfig.java
create mode 100644 src/main/java/fr/osug/doi/domain/DoiCommon.java
create mode 100644 src/main/java/fr/osug/doi/service/DataciteClient.java
diff --git a/doc/application.yml b/doc/application.yml
index c05c9de..125b5cf 100644
--- a/doc/application.yml
+++ b/doc/application.yml
@@ -6,6 +6,7 @@ osug:
doi:
prefix: 10.17178
domain: http://doi.osug.fr
+ dataciteClientEnabled: true
datacite:
user: INIST.OSUG
diff --git a/src/main/java/fr/osug/DOIApplication.java b/src/main/java/fr/osug/DOIApplication.java
index dc87d99..63a7bc9 100644
--- a/src/main/java/fr/osug/DOIApplication.java
+++ b/src/main/java/fr/osug/DOIApplication.java
@@ -6,6 +6,7 @@ package fr.osug;
import ch.qos.logback.classic.Level;
import fr.osug.doi.ProcessPipeline;
import fr.osug.doi.DOIConfig;
+import fr.osug.doi.GeneratePipeline;
import fr.osug.doi.Paths;
import fr.osug.doi.ProjectConfig;
import java.io.IOException;
@@ -24,11 +25,14 @@ import org.springframework.boot.ApplicationRunner;
import org.springframework.boot.ExitCodeGenerator;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.autoconfigure.web.EmbeddedServletContainerAutoConfiguration;
+import org.springframework.boot.autoconfigure.web.WebMvcAutoConfiguration;
import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Component;
-import org.springframework.transaction.annotation.Transactional;
+import org.springframework.transaction.annotation.EnableTransactionManagement;
-@SpringBootApplication
+@SpringBootApplication(exclude = {EmbeddedServletContainerAutoConfiguration.class, WebMvcAutoConfiguration.class})
+@EnableTransactionManagement
public class DOIApplication {
private final static Logger logger = LoggerFactory.getLogger(DOIApplication.class.getName());
@@ -117,6 +121,9 @@ public class DOIApplication {
case "process":
doProcess(aa);
break;
+ case "generate":
+ doGenerate(aa);
+ break;
default:
if (action != null) {
fail("Unsupported action: " + action);
@@ -124,22 +131,17 @@ public class DOIApplication {
}
}
-// @Transactional
- public void doProcess(ApplicationArguments aa) {
-
+ private void doProcess(ApplicationArguments aa) {
// Required values:
final String project = getRequiredValue("project", aa);
-
+ // Optional values:
final boolean saveCSV = aa.containsOption("csv");
try {
- final ProjectConfig projectConfig = new ProjectConfig(project);
+ final ProjectConfig projectConfig = new ProjectConfig(doiConfig.getPathConfig(), project);
projectConfig.setSaveCSV(saveCSV);
- final ProcessPipeline pipeline = new ProcessPipeline(
- doiConfig,
- projectConfig);
-
+ final ProcessPipeline pipeline = new ProcessPipeline(doiConfig, projectConfig);
pipeline.process();
} catch (IOException ioe) {
@@ -148,20 +150,40 @@ public class DOIApplication {
}
}
+ private void doGenerate(ApplicationArguments aa) {
+ // Optional values:
+ final String project = getOptionalValue("project", aa);
+ // mode (staging / public)
+ final String mode = getOptionalValue("mode", aa);
+
+ final boolean doStaging = (mode == null) || "staging".equalsIgnoreCase(mode) || "all".equalsIgnoreCase(mode);
+ final boolean doPublic = "public".equalsIgnoreCase(mode) || "all".equalsIgnoreCase(mode);
+
+ try {
+ final GeneratePipeline pipeline = new GeneratePipeline(doiConfig, doStaging, doPublic);
+ pipeline.process(project);
+
+ } catch (IOException ioe) {
+ // will rollback transaction:
+ throw new RuntimeException(ioe);
+ }
+ }
+
/** Show command arguments help. */
public static void showArgumentsHelp() {
System.out.println("---------------------------------- Arguments help ------------------------------");
System.out.println("| Key Value Description |");
System.out.println("|------------------------------------------------------------------------------|");
- System.out.println("| --action=[help|process] Define the command to perform |");
+ System.out.println("| --action=[help|process|generate] Define the command to perform |");
System.out.println("| [--v=0|1|2|3|4|5] Define console logging level |");
System.out.println("| |");
System.out.println("| LOG LEVELS : 0 = OFF, 1 = ERROR, 2 = WARNING, 3 = INFO, 4 = DEBUG, 5 = ALL |");
System.out.println("|------------------------------------------------------------------------------|");
System.out.println("| Action [process]: |");
- System.out.println("| --project=<dir> Project folder name (case sensitive) |");
+ System.out.println("| --project=<name> Project folder name (case sensitive) |");
System.out.println("| [--csv] CSV Output format |");
- System.out.println("| [--xml] XML Output format (default) |");
+ System.out.println("| Action [generate]: |");
+ System.out.println("| --project=<name> Optional Project folder name (case sensitive) |");
System.out.println("|------------------------------------------------------------------------------|\n");
}
@@ -180,7 +202,14 @@ public class DOIApplication {
this.exitCode = exitCode;
}
- private String getRequiredValue(String opt, final ApplicationArguments aa) {
+ private String getOptionalValue(final String opt, final ApplicationArguments aa) {
+ if (aa.containsOption(opt)) {
+ return getSingleValue(opt, aa);
+ }
+ return null;
+ }
+
+ private String getRequiredValue(final String opt, final ApplicationArguments aa) {
if (!aa.containsOption(opt)) {
fail("Missing argument '" + opt + "' !");
return null;
@@ -188,7 +217,7 @@ public class DOIApplication {
return getSingleValue(opt, aa);
}
- private static String getSingleValue(String opt, final ApplicationArguments aa) {
+ private static String getSingleValue(final String opt, final ApplicationArguments aa) {
final List<String> values = aa.getOptionValues(opt);
if (values.size() != 1) {
throw new IllegalArgumentException("Too many values for argument '" + opt + "': " + values);
diff --git a/src/main/java/fr/osug/doi/Const.java b/src/main/java/fr/osug/doi/Const.java
index 9ba607d..0e44f5d 100644
--- a/src/main/java/fr/osug/doi/Const.java
+++ b/src/main/java/fr/osug/doi/Const.java
@@ -5,7 +5,6 @@ package fr.osug.doi;
/**
*
- * @author bourgesl
*/
public interface Const {
diff --git a/src/main/java/fr/osug/doi/CsvData.java b/src/main/java/fr/osug/doi/CsvData.java
index 571b5cd..940c82c 100644
--- a/src/main/java/fr/osug/doi/CsvData.java
+++ b/src/main/java/fr/osug/doi/CsvData.java
@@ -9,7 +9,6 @@ import org.slf4j.LoggerFactory;
/**
* Generic CSV Data holder
- * @author bourgesl
*/
public class CsvData {
diff --git a/src/main/java/fr/osug/doi/CsvUtil.java b/src/main/java/fr/osug/doi/CsvUtil.java
index afdfa5e..7c98c13 100644
--- a/src/main/java/fr/osug/doi/CsvUtil.java
+++ b/src/main/java/fr/osug/doi/CsvUtil.java
@@ -7,23 +7,22 @@ import com.opencsv.CSVReader;
import com.opencsv.CSVWriter;
import fr.osug.util.FileUtils;
import java.io.File;
-import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FilenameFilter;
import java.io.IOException;
-import java.io.InputStreamReader;
import java.io.OutputStream;
-import java.io.OutputStreamWriter;
-import java.io.Reader;
-import java.io.Writer;
-import java.nio.charset.StandardCharsets;
+import java.net.MalformedURLException;
+import java.net.URL;
import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
*
- * @author bourgesl
*/
public final class CsvUtil {
@@ -36,7 +35,7 @@ public final class CsvUtil {
}
public static CsvData read(final File csvFile) throws IOException {
- final CSVReader reader = new CSVReader(getTextReader(csvFile), Const.SEPARATOR);
+ final CSVReader reader = new CSVReader(FileUtils.getTextReader(csvFile), Const.SEPARATOR);
return new CsvData(csvFile.getAbsolutePath(), reader.readAll());
}
@@ -48,7 +47,7 @@ public final class CsvUtil {
}
public static void write(final CsvData data, final OutputStream out) throws IOException {
- final CSVWriter writer = new CSVWriter(getTextWriter(out), Const.SEPARATOR);
+ final CSVWriter writer = new CSVWriter(FileUtils.getTextWriter(out), Const.SEPARATOR);
try {
writer.writeAll(data.getRows(), false);
} finally {
@@ -56,22 +55,14 @@ public final class CsvUtil {
}
}
- public static Reader getTextReader(final File file) throws IOException {
- return new InputStreamReader(new FileInputStream(file), StandardCharsets.UTF_8);
- }
-
- public static Writer getTextWriter(final File file) throws IOException {
- return getTextWriter(new FileOutputStream(file));
- }
-
- public static Writer getTextWriter(final OutputStream out) throws IOException {
- return new OutputStreamWriter(out, StandardCharsets.UTF_8);
- }
-
public static File[] findCSVFiles(final String dirPath) throws IOException {
final File inputDir = FileUtils.getRequiredDirectory(dirPath).getCanonicalFile();
- // find CSV files:
- return CsvUtil.getSortedFiles(inputDir, ".*\\.csv");
+ return findCSVFiles(inputDir);
+ }
+
+ public static File[] findCSVFiles(final File dir) throws IOException {
+ logger.debug("findCSVFiles: {}", dir);
+ return CsvUtil.getSortedFiles(dir, ".*\\.csv");
}
public static File[] getSortedFiles(final File directory, final String regexp) {
@@ -87,4 +78,40 @@ public final class CsvUtil {
return dataFiles;
}
+
+ public static Map<String, String> loadUrlMapping(final File csvFile) throws IOException {
+ Map<String, String> urlMapping = Collections.emptyMap();
+
+ if (csvFile.exists()) {
+ final CsvData data = CsvUtil.read(csvFile);
+ logger.debug("CSV data: {}", data);
+
+ final List<String[]> rows = data.getRows();
+
+ urlMapping = new LinkedHashMap<String, String>(rows.size());
+
+ // Get mappings [DOI|URL]
+ for (String[] cols : rows) {
+ if (cols == null || cols.length != 2) {
+ continue;
+ }
+ String key = cols[0].trim();
+ if (key.isEmpty() || key.charAt(0) == Const.COMMENT) {
+ continue;
+ }
+ String url = cols[1].trim();
+ if (url.isEmpty() || !url.startsWith("http")) {
+ continue;
+ }
+ try {
+ // should detect repeated keys or values ?
+ urlMapping.put(key, new URL(url).toString());
+ } catch (MalformedURLException mue) {
+ logger.info("invalid URL: {}", url);
+ }
+ }
+ }
+ logger.debug("urlMapping: {}", urlMapping);
+ return urlMapping;
+ }
}
diff --git a/src/main/java/fr/osug/doi/DOIConfig.java b/src/main/java/fr/osug/doi/DOIConfig.java
index 49b71bc..5516897 100644
--- a/src/main/java/fr/osug/doi/DOIConfig.java
+++ b/src/main/java/fr/osug/doi/DOIConfig.java
@@ -3,9 +3,11 @@
******************************************************************************/
package fr.osug.doi;
+import fr.osug.doi.service.DataciteClient;
import fr.osug.doi.service.DoiService;
import fr.osug.xml.XmlFactory;
import fr.osug.xml.validator.XmlValidatorFactory;
+import java.io.IOException;
import java.util.Arrays;
import javax.annotation.PostConstruct;
import org.slf4j.Logger;
@@ -14,22 +16,16 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.Configuration;
-import org.springframework.transaction.annotation.EnableTransactionManagement;
/**
*
- * @author bourgesl
*/
@Configuration
@ConfigurationProperties(prefix = "osug.doi")
-@EnableTransactionManagement
public class DOIConfig {
private final static Logger logger = LoggerFactory.getLogger(DOIConfig.class.getName());
- /* debug flag */
- private boolean debug = false;
-
@Autowired
private ApplicationContext appCtx;
@@ -40,20 +36,43 @@ public class DOIConfig {
@Autowired
private DoiService doiService;
-
+
+ @Autowired
+ private DataciteConfig dataciteConfig;
+
+ /** path config */
+ private PathConfig pathConfig;
+
+ /** debug flag */
+ private boolean debug = false;
+
/** DOI Prefix (registered in datacite) */
private String prefix;
/** DOI Domain (registered in datacite) */
private String domain;
+ /** enable/disable the Datacite API client (REST) */
+ private boolean dataciteClientEnabled;
+
@PostConstruct
public void initialize() {
+ try {
+ this.pathConfig = new PathConfig(Paths.DIR_WEB);
+ } catch (IOException ioe) {
+ throw new IllegalStateException("Invalid paths: ", ioe);
+ }
+
this.debug = Arrays.asList(appCtx.getEnvironment().getActiveProfiles()).contains("debug");
logger.info("debug mode: {}", debug);
logger.info("doi prefix: {}", prefix);
logger.info("doi domain: {}", domain);
+ logger.info("Enable Datacite Client: {}", isDataciteClientEnabled());
+ }
+
+ public PathConfig getPathConfig() {
+ return pathConfig;
}
public boolean isDebug() {
@@ -76,6 +95,10 @@ public class DOIConfig {
return doiService;
}
+ public DataciteClient getDataciteClient() {
+ return dataciteConfig.getClient();
+ }
+
public String getPrefix() {
return prefix;
}
@@ -92,4 +115,12 @@ public class DOIConfig {
this.domain = domain;
}
+ public boolean isDataciteClientEnabled() {
+ return dataciteClientEnabled;
+ }
+
+ public void setDataciteClientEnabled(final boolean dataciteClientEnabled) {
+ this.dataciteClientEnabled = dataciteClientEnabled;
+ }
+
}
diff --git a/src/main/java/fr/osug/doi/DataciteConfig.java b/src/main/java/fr/osug/doi/DataciteConfig.java
new file mode 100644
index 0000000..383c0c0
--- /dev/null
+++ b/src/main/java/fr/osug/doi/DataciteConfig.java
@@ -0,0 +1,60 @@
+/*******************************************************************************
+ * OSUG-DOI project ( http://doi.osug.fr ) - Copyright (C) CNRS.
+ ******************************************************************************/
+package fr.osug.doi;
+
+import fr.osug.doi.service.DataciteClient;
+import javax.annotation.PostConstruct;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.boot.web.client.RestTemplateBuilder;
+import org.springframework.context.annotation.Configuration;
+
+/**
+ *
+ */
+@Configuration
+@ConfigurationProperties(prefix = "datacite")
+public class DataciteConfig {
+
+ private final static Logger logger = LoggerFactory.getLogger(DataciteConfig.class.getName());
+
+ /** User account (registered in datacite) */
+ private String user;
+
+ /** User password (registered in datacite) */
+ private String password;
+
+ @Autowired
+ private RestTemplateBuilder restTemplateBuilder;
+
+ private DataciteClient dcClient;
+
+ @PostConstruct
+ public void initialize() {
+ logger.info("Datacite user: [{}]", user);
+ if (user != null) {
+ restTemplateBuilder = restTemplateBuilder.basicAuthorization(user, password);
+ }
+ dcClient = new DataciteClient(restTemplateBuilder);
+ }
+
+ public String getUser() {
+ return user;
+ }
+
+ public void setUser(String user) {
+ this.user = user;
+ }
+
+ public void setPassword(String password) {
+ this.password = password;
+ }
+
+ public DataciteClient getClient() {
+ return dcClient;
+ }
+
+}
diff --git a/src/main/java/fr/osug/doi/DoiCsvData.java b/src/main/java/fr/osug/doi/DoiCsvData.java
index 60a818d..2284a21 100644
--- a/src/main/java/fr/osug/doi/DoiCsvData.java
+++ b/src/main/java/fr/osug/doi/DoiCsvData.java
@@ -16,7 +16,6 @@ import java.util.Set;
/**
*
- * @author bourgesl
*/
public final class DoiCsvData extends CsvData {
diff --git a/src/main/java/fr/osug/doi/DoiCsvSplitter.java b/src/main/java/fr/osug/doi/DoiCsvSplitter.java
index d60796f..7e416ae 100644
--- a/src/main/java/fr/osug/doi/DoiCsvSplitter.java
+++ b/src/main/java/fr/osug/doi/DoiCsvSplitter.java
@@ -10,7 +10,6 @@ import java.util.List;
/**
*
- * @author bourgesl
*/
public class DoiCsvSplitter {
diff --git a/src/main/java/fr/osug/doi/DoiCsvToXml.java b/src/main/java/fr/osug/doi/DoiCsvToXml.java
index ab9c6b4..7d7d2e4 100644
--- a/src/main/java/fr/osug/doi/DoiCsvToXml.java
+++ b/src/main/java/fr/osug/doi/DoiCsvToXml.java
@@ -3,9 +3,6 @@
******************************************************************************/
package fr.osug.doi;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import org.slf4j.Logger;
@@ -13,7 +10,6 @@ import org.slf4j.LoggerFactory;
/**
*
- * @author bourgesl
*/
public class DoiCsvToXml {
@@ -85,20 +81,6 @@ public class DoiCsvToXml {
}
}
- public static void writeXml(final String content, final File xmlFile) throws IOException {
- logger.info("Writing: {}", xmlFile.getAbsolutePath());
-
- BufferedWriter writer = null;
- try {
- writer = new BufferedWriter(CsvUtil.getTextWriter(xmlFile), content.length());
- writer.write(content);
- } finally {
- if (writer != null) {
- writer.close();
- }
- }
- }
-
public static void xmlEscapeText(final String value, final StringBuilder sb) {
if (value != null) {
for (int i = 0, len = value.length(); i < len; i++) {
diff --git a/src/main/java/fr/osug/doi/DoiTemplates.java b/src/main/java/fr/osug/doi/DoiTemplates.java
index 04d9740..7d91818 100644
--- a/src/main/java/fr/osug/doi/DoiTemplates.java
+++ b/src/main/java/fr/osug/doi/DoiTemplates.java
@@ -12,7 +12,6 @@ import org.slf4j.LoggerFactory;
/**
*
- * @author bourgesl
*/
public final class DoiTemplates {
diff --git a/src/main/java/fr/osug/doi/GeneratePipeline.java b/src/main/java/fr/osug/doi/GeneratePipeline.java
index bf3fbb1..601fe15 100644
--- a/src/main/java/fr/osug/doi/GeneratePipeline.java
+++ b/src/main/java/fr/osug/doi/GeneratePipeline.java
@@ -7,51 +7,37 @@ import fr.osug.doi.domain.Doi;
import fr.osug.doi.domain.DoiStaging;
import fr.osug.doi.domain.Project;
import fr.osug.doi.domain.IndexEntry;
-import fr.osug.doi.domain.NameEntry;
import fr.osug.doi.repository.DoiStagingRepository;
import fr.osug.doi.repository.ProjectRepository;
import fr.osug.doi.service.DoiService;
-import fr.osug.doi.validation.ValidationUtil;
import fr.osug.util.FileUtils;
-import fr.osug.xml.validator.ErrorMessage;
-import fr.osug.xml.validator.ValidationResult;
-import fr.osug.xml.validator.XmlValidator;
import java.io.File;
-import java.io.FileInputStream;
import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
-import java.util.Date;
import java.util.HashMap;
-import java.util.LinkedHashSet;
import java.util.List;
-import java.util.ListIterator;
import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
import javax.xml.transform.stream.StreamResult;
import javax.xml.transform.stream.StreamSource;
-import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.springframework.transaction.annotation.Transactional;
+import fr.osug.doi.domain.DoiCommon;
+import fr.osug.doi.repository.DoiBaseRepository;
+import java.util.Arrays;
/**
*
- * @author bourgesl
*/
-public final class ProcessPipeline {
+public class GeneratePipeline {
- private final static Logger logger = LoggerFactory.getLogger(ProcessPipeline.class.getName());
+ private final static Logger logger = LoggerFactory.getLogger(GeneratePipeline.class.getName());
public final static String HTML_EXT = ".html";
public final static String HTML_INDEX = "index" + HTML_EXT;
public final static String HTML_REPORT = "report" + HTML_EXT;
public final static String HTML_ERROR = "error" + HTML_EXT;
- public final static String XSD_DOI_SCHEMA = "file://" + Paths.DIR_XSD_SCHEMA + "/metadata.xsd";
-
- private final static String XSLT_XML_TO_DOI = "xml2doi.xsl";
private final static String XSLT_DOI_TO_LANDING_PAGE = "doi2landing.xsl";
private final static String XSLT_PROJECT_INDEX = "project_index.xsl";
@@ -78,54 +64,93 @@ public final class ProcessPipeline {
private final static String XSLT_PARAM_CURRENT = "current";
// members
- private Date _now = null;
private final DOIConfig doiConfig;
- private final ProjectConfig projectConfig;
- // temporary data
- private final PipelineGlobalData globalData = new PipelineGlobalData();
+ private final boolean doStaging;
+ private final boolean doPublic;
+ // temporary data (process pipeline)
+ private final PipelineCommonData globalData;
+ /** child pipeline */
+ private final GenerateRedirectPipeline pipe;
+
+ public GeneratePipeline(final DOIConfig doiConfig,
+ final boolean doStaging,
+ final boolean doPublic) {
+ this(doiConfig, doStaging, doPublic, null);
+ }
- public ProcessPipeline(final DOIConfig doiConfig,
- final ProjectConfig projectConfig) throws IOException {
+ GeneratePipeline(final DOIConfig doiConfig,
+ final boolean doStaging,
+ final boolean doPublic,
+ final PipelineCommonData globalData) {
this.doiConfig = doiConfig;
- this.projectConfig = projectConfig;
+ this.doStaging = doStaging;
+ this.doPublic = doPublic;
+ this.globalData = (globalData != null) ? globalData : new PipelineCommonData();
+ // child pipeline:
+ this.pipe = new GenerateRedirectPipeline(doiConfig);
}
- public void process() throws IOException {
+ public void process(final String project) throws IOException {
+ logger.info("doStaging: {}", doStaging);
+ logger.info("doPublic: {}", doPublic);
- // 1 - Process complete datasets (csv):
- for (File inputCsvFile : projectConfig.getMetadataCsvFiles()) {
- processCsvFile(inputCsvFile, false);
- }
+ final DoiService doiService = doiConfig.getDoiService();
+ final ProjectRepository pr = doiService.getProjectRepository();
- // 2 - Process partial datasets (csv to be merged with templates):
- for (File inputCsvFile : projectConfig.getInputCsvFiles()) {
- processCsvFile(inputCsvFile, true);
+ // all or project by project:
+ final List<String> projectNames;
+ if (project == null) {
+ projectNames = pr.findAllNamesOrderByNameAsc();
+ } else {
+ projectNames = Arrays.asList(new String[]{project});
}
- // TODO: 3 - process complete datasets as XML files (full metadata)
- // global validation of references
- validateRefs();
+ logger.info("projectNames: {}", projectNames);
- // 4 - Landing pages
- generateLandingPages();
+ // 1 - Landing pages for selected project(s):
+ for (String projectName : projectNames) {
+ // get existing project:
+ final Project p = pr.findOneByName(projectName);
- // TODO: test publication datacite ?
- // Update database:
- updateDatabase();
+ if (p == null) {
+ logger.warn("Unknown Project [{}]", projectName);
+ } else {
+ final ProjectConfig projectConfig = new ProjectConfig(doiConfig.getPathConfig(), p.getName());
+ generateLandingPages(projectConfig);
+ }
+ }
- // Update index pages:
- generateIndexPagesStaging();
+ // 2 - Update index pages:
+ generateIndexPages();
}
- private void generateLandingPages() throws IOException {
-
- final File webStaging = projectConfig.getWebStagingProjectDir();
- logger.info("Generating landing pages into {}", webStaging);
+ public void generateLandingPages(final ProjectConfig projectConfig) throws IOException {
+ if (doStaging) {
+ generateLandingPages(projectConfig, true);
+ }
+ if (doPublic) {
+ generateLandingPages(projectConfig, false);
+ }
+ }
- final File webStagingEmbed = projectConfig.getWebStagingProjectEmbedDir();
- final File webStagingXml = projectConfig.getWebStagingProjectXmlDir();
+ private void generateLandingPages(final ProjectConfig projectConfig,
+ final boolean isStaging) throws IOException {
+ final File webDir;
+ final File webEmbedDir;
+ final File webXmlDir;
+ if (isStaging) {
+ webDir = projectConfig.getWebStagingProjectDir();
+ webEmbedDir = projectConfig.getWebStagingProjectEmbedDir();
+ webXmlDir = projectConfig.getWebStagingProjectXmlDir();
+ } else {
+ webDir = projectConfig.getWebPublicProjectDir();
+ webEmbedDir = projectConfig.getWebPublicProjectEmbedDir();
+ webXmlDir = projectConfig.getWebPublicProjectXmlDir();
+ }
+ logger.info("Generating landing pages into {}", webDir);
+ // Note: these fields should be in project (db) to avoid inconsistency accross staging then public runs:
final File dataAccessFileLocation = new File(projectConfig.getProjectConf(), CONFIG_ACCESS_INSTRUCTIONS);
final File dataAccessFile = FileUtils.getFile(dataAccessFileLocation);
if (dataAccessFile == null) {
@@ -134,127 +159,198 @@ public final class ProcessPipeline {
// TODO: use mapping file [DOI:dataAccessUrl] like url mapping:
final String defaultDataAccessUrl = projectConfig.getProperty(ProjectConfig.CONF_KEY_DATA_ACCESS_URL);
+ if (defaultDataAccessUrl == null) {
+ logger.warn("Missing default data access url");
+ }
final Map<String, Object> xslParameters = new HashMap<String, Object>(8);
- xslParameters.put(XSLT_PARAM_DATE, now().toString());
+ xslParameters.put(XSLT_PARAM_DATE, globalData.now.toString());
if (dataAccessFile != null) {
xslParameters.put(XSLT_PARAM_DATA_ACCESS_HTML, dataAccessFile.toURI().toString());
}
- if (defaultDataAccessUrl != null) {
- xslParameters.put(XSLT_PARAM_DATA_ACCESS_URL, defaultDataAccessUrl);
- }
- for (PipelineDoiData doiData : globalData.getDoiDatas().values()) {
- final String doiSuffix = doiData.getDoiSuffix();
- logger.info("processing {}", doiSuffix);
+ if (globalData instanceof ProcessPipelineData) {
+ for (ProcessPipelineDoiData doiData : ((ProcessPipelineData) globalData).getDoiDatas().values()) {
+ final String doiSuffix = doiData.getDoiSuffix();
+ logger.info("processing {}", doiSuffix);
- // Get external Landing page URL:
- final String landingPageUrl = projectConfig.getLandingPageUrl(doiSuffix);
- if (landingPageUrl != null) {
- logger.info("landingPageUrl: {}", landingPageUrl);
- xslParameters.put(XSLT_PARAM_LANDING_PAGE_URL, landingPageUrl);
+ // Get specific data access URL:
+ String dataAccessUrl = projectConfig.getUrlDataAccess(doiSuffix);
+ if (dataAccessUrl == null) {
+ dataAccessUrl = defaultDataAccessUrl;
+ }
+ if (dataAccessUrl != null) {
+ logger.debug("dataAccessUrl: {}", dataAccessUrl);
+ doiData.setDataAccessUrl(dataAccessUrl);
+ xslParameters.put(XSLT_PARAM_DATA_ACCESS_URL, dataAccessUrl);
+ } else {
+ xslParameters.remove(XSLT_PARAM_DATA_ACCESS_URL);
+ }
- doiData.setLandingPageUrl(landingPageUrl);
- } else {
- xslParameters.remove(XSLT_PARAM_LANDING_PAGE_URL);
- }
+ // Get external Landing page URL:
+ final String landingPageExtUrl = projectConfig.getUrlLandingPage(doiSuffix);
+ if (landingPageExtUrl != null) {
+ logger.debug("landingPageUrl: {}", landingPageExtUrl);
+ doiData.setLandingPageExtUrl(landingPageExtUrl);
+ xslParameters.put(XSLT_PARAM_LANDING_PAGE_URL, landingPageExtUrl);
+ } else {
+ xslParameters.remove(XSLT_PARAM_LANDING_PAGE_URL);
+ }
- // add warnings:
- if (dataAccessFile == null) {
- doiData.addError("Missing data access instructions (" + CONFIG_ACCESS_INSTRUCTIONS + ")");
- }
- if (StringUtils.isEmpty(defaultDataAccessUrl)) {
- doiData.addError("Missing data access url");
+ // add warnings:
+ if (dataAccessFile == null) {
+ doiData.addError("Missing data access instructions (" + CONFIG_ACCESS_INSTRUCTIONS + ")");
+ }
+ if (dataAccessUrl == null) {
+ doiData.addError("Missing data access url");
+ }
+
+ final File stagingFileDOI = doiData.getStagingFileDOI();
+
+ if (stagingFileDOI == null) {
+ logger.info("Ignoring {} (no xml)", doiSuffix);
+ } else {
+ final String landingPageFilePath = generateLandingPages(projectConfig, stagingFileDOI, xslParameters, webDir, webEmbedDir, webXmlDir);
+
+ // Update landing local url (staging):
+ doiData.setLandingLocUrl(extractUrlPath(landingPageFilePath));
+ }
}
+ } else {
+ // generate from DB:
- final File stagingFileDOI = doiData.getStagingFileDOI();
+ // Get Public|Staging directory:
+ final File doiDir = (isStaging) ? projectConfig.getStagingDir() : projectConfig.getPublicDir();
- if (stagingFileDOI == null) {
- logger.info("Ignoring {} (no xml)", doiSuffix);
- } else {
- final String filenameNoExt = FileUtils.getFileNameWithoutExtension(stagingFileDOI);
+ final String projectName = projectConfig.getProjectName();
- // main landing page:
- xslParameters.remove(XSLT_PARAM_EMBEDDED);
- xslParameters.put(XSLT_PARAM_WEB_ROOT, "../../");
- File output = new File(webStaging, filenameNoExt + HTML_EXT);
- generateLandingPage(xslParameters, stagingFileDOI, output);
-
- if (projectConfig.getPropertyBoolean(ProjectConfig.CONF_KEY_GENERATE_EMBEDDED)) {
- // embedded mode:
- xslParameters.put(XSLT_PARAM_WEB_ROOT, "../../../");
-
- // embedded mode - full:
- output = new File(webStagingEmbed, filenameNoExt + HTML_EXT);
- xslParameters.put(XSLT_PARAM_EMBEDDED, XSLT_PARAM_EMBEDDED_FULL);
- generateLandingPage(xslParameters, stagingFileDOI, output);
-
- // embedded mode - header:
- output = new File(webStagingEmbed, filenameNoExt + "-header" + HTML_EXT);
- xslParameters.put(XSLT_PARAM_EMBEDDED, XSLT_PARAM_EMBEDDED_HEADER);
- generateLandingPage(xslParameters, stagingFileDOI, output);
-
- // embedded mode - meta:
- output = new File(webStagingEmbed, filenameNoExt + "-meta" + HTML_EXT);
- xslParameters.put(XSLT_PARAM_EMBEDDED, XSLT_PARAM_EMBEDDED_META);
- generateLandingPage(xslParameters, stagingFileDOI, output);
+ final DoiService doiService = doiConfig.getDoiService();
+ final DoiBaseRepository<?> dbr = doiService.getDoiCommonRepository(isStaging);
+
+ final List<? extends DoiCommon> dList = dbr.findByProject(projectName);
+
+ logger.debug("DoiCommon list for project[{}]: {}", projectName, dList);
+
+ for (DoiCommon d : dList) {
+ logger.debug("DoiCommon: {}", d);
+ final Doi doi = d.getDoi();
+
+ final String doiSuffix = doi.getIdentifier();
+ logger.info("processing {}", doiSuffix);
+
+ // Get specific data access URL:
+ final String dataAccessUrl = doi.getDataAccessUrl();
+ if (dataAccessUrl != null) {
+ logger.debug("dataAccessUrl: {}", dataAccessUrl);
+ xslParameters.put(XSLT_PARAM_DATA_ACCESS_URL, dataAccessUrl);
+ } else {
+ xslParameters.remove(XSLT_PARAM_DATA_ACCESS_URL);
+ }
+
+ // Get external Landing page URL:
+ final String landingPageUrl = doi.getLandingExtUrl();
+ if (landingPageUrl != null) {
+ logger.debug("landingPageUrl: {}", landingPageUrl);
+ xslParameters.put(XSLT_PARAM_LANDING_PAGE_URL, landingPageUrl);
+ } else {
+ xslParameters.remove(XSLT_PARAM_LANDING_PAGE_URL);
}
- // Xml DOI:
- output = new File(webStagingXml, stagingFileDOI.getName());
- FileUtils.copy(stagingFileDOI, output);
+ final File stagingFileDOI = new File(doiDir, doiSuffix + Const.FILE_EXT_XML);
+
+ if (!stagingFileDOI.exists()) {
+ logger.warn("Missing {} (no xml)", stagingFileDOI);
+ } else {
+ generateLandingPages(projectConfig, stagingFileDOI, xslParameters, webDir, webEmbedDir, webXmlDir);
+ }
}
}
+ logger.info("Generating landing pages: done");
}
- private void generateLandingPage(final Map<String, Object> xslParameters,
- final File stagingFileDOI, final File output) {
-
- logger.info("generateLandingPage: {}", output);
+ private String generateLandingPages(final ProjectConfig projectConfig,
+ final File xmlFileDOI,
+ final Map<String, Object> xslParameters,
+ final File webDir,
+ final File webEmbedDir,
+ final File webXmlDir) throws IOException {
+
+ final String filenameNoExt = FileUtils.getFileNameWithoutExtension(xmlFileDOI);
+
+ // main landing page:
+ // [public/staging]/PROJECT/DOI_SUFFIX.html
+ xslParameters.remove(XSLT_PARAM_EMBEDDED);
+ xslParameters.put(XSLT_PARAM_WEB_ROOT, "../../");
+ File outputFile = new File(webDir, filenameNoExt + HTML_EXT);
+ generateLandingPage(xslParameters, xmlFileDOI, outputFile);
+
+ // Store reference of the main landing page:
+ final String landingPageFilePath = outputFile.getAbsolutePath();
+
+ if (projectConfig.getPropertyBoolean(ProjectConfig.CONF_KEY_GENERATE_EMBEDDED)) {
+ // embedded mode:
+ xslParameters.put(XSLT_PARAM_WEB_ROOT, "../../../");
+
+ // embedded mode - full:
+ // [public/staging]/PROJECT/embed/DOI_SUFFIX[.html]
+ outputFile = new File(webEmbedDir, filenameNoExt + HTML_EXT);
+ xslParameters.put(XSLT_PARAM_EMBEDDED, XSLT_PARAM_EMBEDDED_FULL);
+ generateLandingPage(xslParameters, xmlFileDOI, outputFile);
+
+ // embedded mode - header:
+ outputFile = new File(webEmbedDir, filenameNoExt + "-header" + HTML_EXT);
+ xslParameters.put(XSLT_PARAM_EMBEDDED, XSLT_PARAM_EMBEDDED_HEADER);
+ generateLandingPage(xslParameters, xmlFileDOI, outputFile);
+
+ // embedded mode - meta:
+ outputFile = new File(webEmbedDir, filenameNoExt + "-meta" + HTML_EXT);
+ xslParameters.put(XSLT_PARAM_EMBEDDED, XSLT_PARAM_EMBEDDED_META);
+ generateLandingPage(xslParameters, xmlFileDOI, outputFile);
+ }
- xslParameters.put(XSLT_PARAM_METADATA_FILE, stagingFileDOI.getName());
- logger.debug("xslParameters: {}", xslParameters);
+ // Xml DOI:
+ outputFile = new File(webXmlDir, xmlFileDOI.getName());
+ FileUtils.copy(xmlFileDOI, outputFile);
- // use XSLT to generate landing page from the doi xml file:
- doiConfig.getXmlFactory().transform(
- new StreamSource(stagingFileDOI),
- Paths.DIR_XSL + XSLT_DOI_TO_LANDING_PAGE,
- xslParameters, true,
- new StreamResult(output)
- );
+ return landingPageFilePath;
}
- @Transactional
- /* public needed by Transactional */
- public void updateDatabase() {
- final String projectName = projectConfig.getProjectName();
+ private void generateLandingPage(final Map<String, Object> xslParameters,
+ final File inputFileDOI, final File outputFile) {
+
+ xslParameters.put(XSLT_PARAM_METADATA_FILE, inputFileDOI.getName());
- final Date now = now();
+ // use XSLT to generate landing page from the doi xml file:
+ transform(inputFileDOI, Paths.DIR_XSL + XSLT_DOI_TO_LANDING_PAGE, xslParameters, outputFile);
+ }
- // Get or create the project:
- final DoiService doiService = doiConfig.getDoiService();
- final Project project = doiService.getOrCreateProject(projectName, now);
-
- // Update Dois:
- for (PipelineDoiData doiData : globalData.getDoiDatas().values()) {
- // Update DOI in staging phase:
- doiService.createOrUpdateStagingDoi(project, doiData.getDoiSuffix(),
- doiData.getTitle(), doiData.getLandingPageUrl(),
- doiData.isValid(), doiData.getMetadataMd5(),
- doiData.messagesToString(), now);
+ public void generateIndexPages() throws IOException {
+ if (doStaging) {
+ generateIndexPages(true);
+ }
+ if (doPublic) {
+ generateIndexPages(false);
}
- // Update Project:
- doiService.updateProjectDate(projectName, now);
+ // generate .htaccess (for apache2)
+ pipe.generateRedirects();
}
- @Transactional
- /* public needed by Transactional */
- public void generateIndexPagesStaging() {
- final File webStaging = projectConfig.getWebStagingDir();
+ private void generateIndexPages(final boolean isStaging) {
+ final String folderType;
+ final File webDir;
+ if (isStaging) {
+ folderType = Paths.DIR_WEB_STAGING;
+ webDir = doiConfig.getPathConfig().getWebStagingDir();
+ } else {
+ folderType = Paths.DIR_WEB_PUBLIC;
+ webDir = doiConfig.getPathConfig().getWebPublicDir();
+ }
+ logger.info("Generating index pages into {}", webDir);
final DoiService doiService = doiConfig.getDoiService();
final ProjectRepository pr = doiService.getProjectRepository();
+ final DoiBaseRepository<?> dbr = doiService.getDoiCommonRepository(isStaging);
final DoiStagingRepository dsr = doiService.getDoiStagingRepository();
final List<Project> projects = pr.findAllByOrderByNameAsc();
@@ -262,430 +358,136 @@ public final class ProcessPipeline {
// Project listing:
List<IndexEntry> entries = new ArrayList<IndexEntry>();
+ // index.html:
for (Project p : projects) {
final String projectName = p.getName();
- final Long countDoi = dsr.countByProject(projectName);
- final IndexEntry ie = new IndexEntry(projectName, p.getDescription(), p.getUpdateDate(), countDoi);
+ final Long countDoi = dbr.countByProject(projectName);
- logger.debug("IndexEntry: {}", ie);
- entries.add(ie);
+ if (countDoi > 0L) {
+ final IndexEntry ie = new IndexEntry(projectName, p.getDescription(), p.getUpdateDate(), countDoi);
+
+ logger.debug("IndexEntry: {}", ie);
+ entries.add(ie);
+ }
}
- // serialize entries and transform in the /staging/index.html
+ // serialize entries and transform in the /[staging|public]/index.html
IndexUtil.write(entries, globalData.buffer);
String xmlDoc = globalData.buffer.toString();
logger.debug("xmlDoc(project index): \n{}", xmlDoc);
final Map<String, Object> xslParameters = new HashMap<String, Object>(8);
- xslParameters.put(XSLT_PARAM_DATE, now().toString());
- xslParameters.put(XSLT_PARAM_CURRENT, "staging");
+ xslParameters.put(XSLT_PARAM_DATE, globalData.now.toString());
+ xslParameters.put(XSLT_PARAM_CURRENT, folderType);
xslParameters.put(XSLT_PARAM_WEB_ROOT, "../");
- File output = new File(webStaging, HTML_INDEX);
+ File outputFile = new File(webDir, HTML_INDEX);
// use XSLT to generate the index page:
- doiConfig.getXmlFactory().transform(
- new StreamSource(new StringReader(xmlDoc)),
- Paths.DIR_XSL_ROOT + XSLT_PROJECT_INDEX,
- xslParameters, true,
- new StreamResult(output)
- );
+ transform(xmlDoc, Paths.DIR_XSL_ROOT + XSLT_PROJECT_INDEX, xslParameters, outputFile);
// DOI listing per project:
for (Project p : projects) {
final String projectName = p.getName();
- final List<DoiStaging> dsList = dsr.findByProject(projectName);
-
- logger.debug("DoiStaging list for project[{}]: {}", projectName, dsList);
- // index.html:
+ // PROJECT/index.html:
entries.clear();
- for (DoiStaging ds : dsList) {
- logger.debug("DoiStaging: {}", ds);
+ final List<? extends DoiCommon> dList = dbr.findByProject(projectName);
- final Doi doi = ds.getDoi();
+ logger.debug("DoiCommon list for project[{}]: {}", projectName, dList);
- final IndexEntry ie = new IndexEntry(doi.getIdentifier(), doi.getDescription(), ds.getUpdateDate());
+ if (!dList.isEmpty()) {
+ for (DoiCommon d : dList) {
+ logger.debug("DoiBase: {}", d);
+ final Doi doi = d.getDoi();
+ final IndexEntry ie = new IndexEntry(doi.getIdentifier(), doi.getDescription(), d.getUpdateDate());
- logger.debug("IndexEntry: {}", ie);
- entries.add(ie);
- }
- // serialize entries and transform in the /staging/PROJECT/index.html
- IndexUtil.write(entries, globalData.buffer);
-
- xmlDoc = globalData.buffer.toString();
- logger.debug("xmlDoc(doi index): \n{}", xmlDoc);
-
- xslParameters.put(XSLT_PARAM_PARENT, "staging");
- xslParameters.put(XSLT_PARAM_CURRENT, projectName);
- xslParameters.put(XSLT_PARAM_WEB_ROOT, "../../");
- // suppose child directory is the project folder:
- output = new File(new File(webStaging, projectName), HTML_INDEX);
-
- // use XSLT to generate the index page:
- doiConfig.getXmlFactory().transform(
- new StreamSource(new StringReader(xmlDoc)),
- Paths.DIR_XSL_ROOT + XSLT_PROJECT_INDEX,
- xslParameters, true,
- new StreamResult(output)
- );
-
- // serialize doi staging list and transform in the /staging/PROJECT/report.html
- IndexUtil.writeReport(dsList, globalData.buffer);
-
- xmlDoc = globalData.buffer.toString();
- logger.debug("xmlDoc(doi report): \n{}", xmlDoc);
-
- xslParameters.put(XSLT_PARAM_PARENT, "staging");
- xslParameters.put(XSLT_PARAM_CURRENT, projectName);
- xslParameters.put(XSLT_PARAM_WEB_ROOT, "../../");
- // suppose child directory is the project folder:
- output = new File(new File(webStaging, projectName), HTML_REPORT);
-
- // use XSLT to generate the report page:
- doiConfig.getXmlFactory().transform(
- new StreamSource(new StringReader(xmlDoc)),
- Paths.DIR_XSL_ROOT + XSLT_PROJECT_REPORT,
- xslParameters, true,
- new StreamResult(output)
- );
-
- // serialize doi staging error list and transform in the /staging/PROJECT/error.html
- final List<DoiStaging> dsErrorList = dsr.findErrorByProject(projectName);
- logger.debug("DoiStaging error list for project[{}]: {}", projectName, dsErrorList);
-
- IndexUtil.writeReport(dsErrorList, globalData.buffer);
-
- xmlDoc = globalData.buffer.toString();
- logger.debug("xmlDoc(error report): \n{}", xmlDoc);
-
- xslParameters.put(XSLT_PARAM_PARENT, "staging");
- xslParameters.put(XSLT_PARAM_CURRENT, projectName);
- xslParameters.put(XSLT_PARAM_WEB_ROOT, "../../");
- // suppose child directory is the project folder:
- output = new File(new File(webStaging, projectName), HTML_ERROR);
-
- // use XSLT to generate the report page:
- doiConfig.getXmlFactory().transform(
- new StreamSource(new StringReader(xmlDoc)),
- Paths.DIR_XSL_ROOT + XSLT_PROJECT_REPORT,
- xslParameters, true,
- new StreamResult(output)
- );
- }
- logger.info("generateIndexPagesStaging: done.");
- }
+ logger.debug("IndexEntry: {}", ie);
+ entries.add(ie);
+ }
+ // serialize entries and transform in the /[staging|public]/PROJECT/index.html
+ IndexUtil.write(entries, globalData.buffer);
- private void validateRefs() throws IOException {
- logger.debug("validateRefs");
+ xmlDoc = globalData.buffer.toString();
+ logger.debug("xmlDoc(doi index): \n{}", xmlDoc);
- final DoiService doiService = doiConfig.getDoiService();
- final String publicPrefix = doiConfig.getPrefix();
-
- for (String doiId : globalData.getDoiIds()) {
- logger.debug("checking refs for [{}]", doiId);
-
- final Set<String> refs = globalData.getDoiRefs(doiId);
- if (refs != null) {
- for (String refId : refs) {
- // Only check DOI references with Prefix Test or Public:
- if (ValidationUtil.isTestPrefix(refId) || refId.startsWith(publicPrefix)) {
- logger.debug("validateRefs: check ref[{}]", refId);
-
- boolean found = false;
-
- // Lookup first in processed identifiers:
- if (globalData.hasDoiId(refId)) {
- found = true;
- } else {
- // Then in database (previous data):
- final String doiSuffix = DoiCsvData.getDoiSuffix(refId);
- if (doiService.getDoi(doiSuffix) != null) {
- found = true;
- }
- }
- if (!found) {
- final PipelineDoiData doiData = globalData.getDoiData(doiId);
- if (doiData != null) {
- doiData.addError("Invalid " + Const.KEY_REL_ID_START + " '" + refId + "' found.");
- }
- }
- }
- }
- }
- }
+ // inherit XSLT_PARAM_DATE:
+ xslParameters.put(XSLT_PARAM_PARENT, folderType);
+ xslParameters.put(XSLT_PARAM_CURRENT, projectName);
+ xslParameters.put(XSLT_PARAM_WEB_ROOT, "../../");
- // Check name references:
- final TreeSet<NameEntry> entries = globalData.getNameRefs();
- if (logger.isDebugEnabled()) {
- logger.debug("name references:\n{}", entries);
- }
+ // suppose child directory is the project folder:
+ final File webProjectDir = new File(webDir, projectName);
+ outputFile = new File(webProjectDir, HTML_INDEX);
- // Collect unique attributes:
- final Set<String> keys = new LinkedHashSet<String>();
- for (NameEntry e : entries) {
- if (e.hasAttributes()) {
- for (String k : e.getAttributes().keySet()) {
- keys.add(k);
- }
- }
- }
+ // use XSLT to generate the PROJECT index page:
+ transform(xmlDoc, Paths.DIR_XSL_ROOT + XSLT_PROJECT_INDEX, xslParameters, outputFile);
- final int nCols = keys.size() + 1;
+ if (isStaging) {
+ @SuppressWarnings("unchecked")
+ final List<DoiStaging> dsList = (List<DoiStaging>) dList;
- final List<String[]> rows = new ArrayList<String[]>();
- // header:
- String[] cols = new String[nCols];
- int i = 0;
- cols[i++] = "#Name";
- for (String k : keys) {
- cols[i++] = k;
- }
- rows.add(cols);
- // data:
- for (NameEntry e : entries) {
- if (e.hasAttributes()) {
- cols = new String[nCols];
- i = 0;
- cols[i++] = e.getName();
- for (String k : keys) {
- cols[i++] = e.getAttribute(k);
- }
- } else {
- cols = new String[]{e.getName()};
- }
- rows.add(cols);
- }
- final CsvData data = new CsvData(rows);
- if (logger.isDebugEnabled()) {
- logger.debug("name reference table:\n{}", data);
- }
+ // serialize doi staging list and transform in the /staging/PROJECT/report.html
+ IndexUtil.writeReport(dsList, globalData.buffer);
- // Save file as CSV:
- final File fileCSV = new File(projectConfig.getTmpDir(), "names" + Const.FILE_EXT_CSV);
- CsvUtil.write(data, fileCSV);
- }
+ xmlDoc = globalData.buffer.toString();
+ logger.debug("xmlDoc(doi report): \n{}", xmlDoc);
- private void processCsvFile(final File inputCsvFile,
- final boolean doMerge) throws IOException {
- /*
- Spliter et produire 1 fichier XML dataCite par jeu.
- Ex: AMMA-CATCH.CL.Run_O.xml
- */
- final List<DoiCsvData> csvDatas = DoiCsvSplitter.split(inputCsvFile);
-
- for (DoiCsvData data : csvDatas) {
- if (logger.isDebugEnabled()) {
- logger.debug("process: data:\n{}", data);
- }
- if (!data.isEmpty()) {
- final String doiId = data.getDoiId();
+ // inherit XSLT_PARAM_DATE, XSLT_PARAM_PARENT, XSLT_PARAM_CURRENT, XSLT_PARAM_WEB_ROOT:
+ // suppose child directory is the project folder:
+ outputFile = new File(webProjectDir, HTML_REPORT);
- // check identifier (missing) ?
- if (ValidationUtil.checkIdentifier(doiId, data)) {
- final String doiSuffix = data.getDoiSuffix();
- logger.info("process: dataset[{}].", doiSuffix);
+ // use XSLT to generate the PROJECT report page:
+ transform(xmlDoc, Paths.DIR_XSL_ROOT + XSLT_PROJECT_REPORT, xslParameters, outputFile);
- final PipelineDoiData doiData = globalData.newDoiData(doiId, doiSuffix, data.getTitle());
+ // serialize doi staging error list and transform in the /staging/PROJECT/error.html
+ final List<DoiStaging> dsErrorList = dsr.findErrorByProject(projectName);
+ logger.debug("DoiStaging error list for project[{}]: {}", projectName, dsErrorList);
- // ignore duplicates
- if (doiData != null) {
- // validate identifier (format)
- ValidationUtil.validateIdentifier(doiId, doiData);
+ IndexUtil.writeReport(dsErrorList, globalData.buffer);
- if (doMerge) {
- mergeCSV(data, doiData);
- }
+ xmlDoc = globalData.buffer.toString();
+ logger.debug("xmlDoc(error report): \n{}", xmlDoc);
- // Validate metadata:
- validateData(data, doiData);
+ // inherit XSLT_PARAM_DATE, XSLT_PARAM_PARENT, XSLT_PARAM_CURRENT, XSLT_PARAM_WEB_ROOT:
+ // suppose child directory is the project folder:
+ outputFile = new File(webProjectDir, HTML_ERROR);
- // Save metadata for dataset:
- saveData(data, doiData);
- }
+ // use XSLT to generate the PROJECT error report page:
+ transform(xmlDoc, Paths.DIR_XSL_ROOT + XSLT_PROJECT_REPORT, xslParameters, outputFile);
}
}
}
+ logger.info("Generating index pages: done");
}
- private void mergeCSV(final DoiCsvData data, final PipelineDoiData doiData) {
-
- final DoiTemplates templates = projectConfig.getTemplates();
-
- /*
- ### 2/ Fusionner avec template observatoire : template_obs.csv
-
- Informations disjointes
- */
- DoiCsvData t = templates.getBase();
- if (t == null) {
- doiData.addError("Missing base template for project '" + projectConfig.getProjectName() + "'");
- } else {
- data.mergeFrom(t);
- }
-
- final String doiId = data.getDoiId(); // not null
-
- /*
- ### 3/ Traitement spécifiques: fusionner avec template pays
-
- Fusionner avec template pays (template_benin.csv):
-
- Trouver le template du pays qui contient geoLocationPlace;Benin
- */
- final String geoLocationPlace = data.getFirstGeoLocationPlace();
- if (geoLocationPlace == null) {
- doiData.addWarning("Missing ''" + Const.KEY_GEO_LOCATION_PLACE + "'' in dataset '" + doiId + "'");
- } else {
- t = templates.getByGeoLocationPlace(geoLocationPlace);
- if (t == null) {
- doiData.addError("Missing country template for '" + geoLocationPlace + "'");
- } else {
- logger.info("Using country template '{}'", geoLocationPlace);
- data.mergeFrom(t);
- }
- }
-
- /*
- ### 4/ Traitement spécifiques: fusionner avec template jeu
-
- Trouver le nom du jeu dans le XML:
- <identifier identifierType="DOI">10.5072/AMMA-CATCH.CL.Run_O</identifier>
-
- Trouver le template du jeu qui contient identifier:DOI;10.5072/AMMA-CATCH.CL.Run_O
- */
- t = templates.getByIdentifier(doiId);
- if (t == null) {
- doiData.addWarning("Missing dataset template for id '" + doiId + "'");
- } else {
- logger.info("Using dataset template for id '{}'", doiId);
- data.mergeFrom(t);
- }
+ private void transform(final File src, final String xslFilePath,
+ final Map<String, Object> xslParameters, final File outputFile) {
+ transform(new StreamSource(src), xslFilePath, xslParameters, outputFile);
}
- private void validateData(final DoiCsvData data, final PipelineDoiData doiData) {
- logger.debug("validateData [{}]", doiData.getDoiId());
- // TODO: check title ?
-
- // check all values:
- final List<String[]> rows = data.getRows();
- for (ListIterator<String[]> it = rows.listIterator(); it.hasNext();) {
- final String[] cols = it.next();
- if (cols != null && cols.length >= 2) {
- if (cols[1].toLowerCase().contains("todo")) {
- doiData.addWarning("TODO detected for key [" + cols[0] + ']');
- }
-
- // Collect name references:
- final String key = cols[0];
- if (key.startsWith(Const.KEY_CREATOR_NAME) || key.startsWith(Const.KEY_CONTRIBUTOR_NAME)) {
- globalData.addNameRef(cols);
- }
- }
- }
-
- final Set<String> refs = data.getReferences();
- if (refs != null) {
- logger.debug("References: {}", refs);
- }
- // anyway: add the doi entry:
- globalData.addRefs(doiData.getDoiId(), refs);
+ private void transform(final String src, final String xslFilePath,
+ final Map<String, Object> xslParameters, final File outputFile) {
+ transform(new StreamSource(new StringReader(src)), xslFilePath, xslParameters, outputFile);
}
- private void saveData(final DoiCsvData data, final PipelineDoiData doiData) throws IOException {
- // Sort keys:
- data.sort();
-
- // Get temporary directory:
- final File tmpDoiDir = projectConfig.getTmpDoiDir();
- // Get Staging directory:
- final File stagingDir = projectConfig.getStagingDir();
-
- // File base name:
- final String baseName = data.getDoiSuffix();
-
- // Save file as XML:
- DoiCsvToXml.write(data, globalData.buffer);
-
- final String xmlDoc = globalData.buffer.toString();
- logger.debug("xmlDoc: \n{}", xmlDoc);
-
- boolean forceSaveCSV = false;
- boolean forceSaveXML = false;
-
- // check non empty file:
- if (xmlDoc.length() == 0) {
- forceSaveCSV = true;
- doiData.addError("No XML output");
- } else {
- globalData.doiDoc.reset();
-
- // use XSLT to write doi xml file:
- doiConfig.getXmlFactory().transform(
- new StreamSource(new StringReader(xmlDoc)),
- Paths.DIR_XSL + XSLT_XML_TO_DOI, true,
- new StreamResult(globalData.doiDoc)); // no xslt-param
-
- final String doiDoc = globalData.doiDoc.toString();
- logger.debug("doiDoc: \n{}", doiDoc);
-
- if (doiDoc.length() == 0) {
- forceSaveXML = true;
- doiData.addError("No Metadata XML output");
- } else {
- final File fileDOI = new File(tmpDoiDir, baseName + Const.FILE_EXT_XML);
- DoiCsvToXml.writeXml(doiDoc, fileDOI);
-
- // compute MD5 on metadata:
- final String xmlMd5 = FileUtils.MD5(new FileInputStream(fileDOI));
- doiData.setMetadataMd5(xmlMd5);
- logger.debug("DOI metadata [{}] MD5: {}", fileDOI, xmlMd5);
-
- // Validate:
- final XmlValidator validator = doiConfig.getXmlValidatorFactory().getInstance(XSD_DOI_SCHEMA);
-
- final ValidationResult vr = new ValidationResult();
- validator.validate(new StreamSource(fileDOI), vr);
-
- logger.info("XSD validation [{}]: {}", fileDOI.getAbsolutePath(), vr.isValid());
-
- if (!vr.isValid()) {
- doiData.addError("Metadata XML validation failed:");
-
- for (ErrorMessage msg : vr.getMessages()) {
- doiData.addError(msg.toString());
- }
- }
-
- // Anyway copy the doi metadata into staging:
- final File stagingFileDOI = new File(stagingDir, fileDOI.getName());
-
- logger.debug("Copy [{}] to [{}]", fileDOI, stagingFileDOI);
+ private void transform(final StreamSource src, final String xslFilePath,
+ final Map<String, Object> xslParameters, final File outputFile) {
- FileUtils.copy(fileDOI, stagingFileDOI);
+ logger.info("generate: {}", outputFile);
- // to be processed at next step:
- doiData.setStagingFileDOI(stagingFileDOI);
- }
- }
-
- // Always save intermediate results in case of failure:
- if (forceSaveCSV || projectConfig.isSaveCSV()) {
- // Save file as CSV:
- final File fileCSV = new File(tmpDoiDir, baseName + Const.FILE_EXT_CSV);
- CsvUtil.write(data, fileCSV);
- }
-
- if (forceSaveXML || doiConfig.isDebug()) {
- final File fileXML = new File(tmpDoiDir, baseName + "-xml" + Const.FILE_EXT_XML);
- DoiCsvToXml.writeXml(xmlDoc, fileXML);
- }
+ doiConfig.getXmlFactory().transform(src, xslFilePath,
+ xslParameters, true, new StreamResult(outputFile)
+ );
}
- private Date now() {
- if (_now == null) {
- _now = new Date();
+ private String extractUrlPath(final String landingPageFilePath) {
+ final String webRootPath = doiConfig.getPathConfig().getWebRootPath();
+ logger.debug("webRootPath: {}", webRootPath);
+
+ if (!landingPageFilePath.startsWith(webRootPath)) {
+ throw new IllegalStateException("Invalid web root path in file: " + landingPageFilePath);
}
- return _now;
+ return landingPageFilePath.substring(webRootPath.length() - 1);
}
}
diff --git a/src/main/java/fr/osug/doi/GenerateRedirectPipeline.java b/src/main/java/fr/osug/doi/GenerateRedirectPipeline.java
new file mode 100644
index 0000000..fd07ddf
--- /dev/null
+++ b/src/main/java/fr/osug/doi/GenerateRedirectPipeline.java
@@ -0,0 +1,208 @@
+/*******************************************************************************
+ * OSUG-DOI project ( http://doi.osug.fr ) - Copyright (C) CNRS.
+ ******************************************************************************/
+package fr.osug.doi;
+
+import fr.osug.doi.domain.Doi;
+import fr.osug.doi.domain.DoiPublic;
+import fr.osug.doi.domain.DoiStaging;
+import fr.osug.doi.domain.Project;
+import fr.osug.doi.domain.Status;
+import fr.osug.doi.repository.DoiPublicRepository;
+import fr.osug.doi.repository.DoiRepository;
+import fr.osug.doi.repository.DoiStagingRepository;
+import fr.osug.doi.repository.ProjectRepository;
+import fr.osug.doi.service.DoiService;
+import fr.osug.util.FileUtils;
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ */
+public class GenerateRedirectPipeline {
+
+ private final static Logger logger = LoggerFactory.getLogger(GenerateRedirectPipeline.class.getName());
+
+ public final static String FILE_HT_ACCESS = ".htaccess";
+
+ // members
+ private final DOIConfig doiConfig;
+ // temporary data (process pipeline)
+ private final PipelineCommonData globalData;
+
+ public GenerateRedirectPipeline(final DOIConfig doiConfig) {
+ this.doiConfig = doiConfig;
+ this.globalData = new PipelineCommonData();
+ }
+
+ public void process() throws IOException {
+ generateRedirects();
+ }
+
+ public void generateRedirects() throws IOException {
+ generateDOIUrls();
+ generateEmbedUrls();
+ }
+
+ private void generateDOIUrls() throws IOException {
+ final StringBuilder sb = prepareBuffer();
+
+ sb.append("\n# Redirect DOI to Landing page:");
+ sb.append("\n# [DOI_SUFFIX] => [landing page Ext]");
+ sb.append("\n# or [/public/<PROJECT>/<DOI_SUFFIX>.html] (public)");
+ sb.append("\n# or [/staging/<PROJECT>/<DOI_SUFFIX>.html] (staging)");
+
+ final DoiService doiService = doiConfig.getDoiService();
+ final ProjectRepository pr = doiService.getProjectRepository();
+ final DoiRepository dr = doiService.getDoiRepository();
+ final DoiPublicRepository dpr = doiService.getDoiPublicRepository();
+ final DoiStagingRepository dsr = doiService.getDoiStagingRepository();
+
+ final List<Project> projects = pr.findAllByOrderByNameAsc();
+
+ // DOI listing per project:
+ for (Project p : projects) {
+ final String projectName = p.getName();
+
+ sb.append("\n# Project ").append(projectName);
+
+ final List<Doi> dois = dr.findByProject(projectName);
+
+ logger.info("Dois for project[{}]: {}", projectName, dois.size());
+
+ for (Doi doi : dois) {
+ final String doiSuffix = doi.getIdentifier();
+
+ if (doi.getStatus() == Status.PUBLIC) {
+ final DoiPublic dp = dpr.findOneByDoi(doi);
+
+ if (dp == null) {
+ logger.warn("Missing DoiPublic for [{}]; skipping Redirect", doiSuffix);
+ } else {
+ sb.append("\nRedirect \"/").append(doiSuffix).append("\" \"");
+
+ if (dp.isActiveExtUrl() && doi.getLandingExtUrl() != null) {
+ /*
+ # [landing page Ext] case:
+ # note: complete pattern match
+ Redirect "/AMMA-CATCH.CE.RainD_Nc" "http://www.amma-catch.org/spip.php?article220"
+ */
+ sb.append(doi.getLandingExtUrl());
+ } else {
+ /*
+ # [/public/<PROJECT>/<DOI_SUFFIX>.html] (public) case:
+ Redirect "/AMMA-CATCH.all" "/public/AMMA-CATCH/AMMA-CATCH.all.html"
+ */
+ sb.append(dp.getLandingLocUrl());
+ }
+ sb.append('\"');
+ }
+ } else {
+ final DoiStaging ds = dsr.findOneByDoi(doi);
+
+ if (ds == null) {
+ logger.warn("Missing DoiStaging for [{}]; skipping Redirect", doiSuffix);
+ } else {
+ sb.append("\nRedirect \"/").append(doiSuffix).append("\" \"");
+
+ // Note: no test on the active flag (public only):
+ if (doi.getLandingExtUrl() != null) {
+ /*
+ # [landing page Ext] case:
+ # note: complete pattern match
+ Redirect "/AMMA-CATCH.CE.RainD_Nc" "http://www.amma-catch.org/spip.php?article220"
+ */
+ sb.append(doi.getLandingExtUrl());
+ } else {
+ /*
+ # [/public/<PROJECT>/<DOI_SUFFIX>.html] (public) case:
+ Redirect "/AMMA-CATCH.all" "/public/AMMA-CATCH/AMMA-CATCH.all.html"
+ */
+ sb.append(ds.getLandingLocUrl());
+ }
+ sb.append('\"');
+ }
+ }
+ }
+ sb.append('\n');
+ } // projects
+
+ final String redirectRules = sb.toString();
+
+ FileUtils.writeFile(redirectRules,
+ new File(doiConfig.getPathConfig().getWebRedirectDir(), FILE_HT_ACCESS));
+
+ logger.debug("generateDOIUrls:\n{}", redirectRules);
+ }
+
+ private void generateEmbedUrls() throws IOException {
+ final StringBuilder sb = prepareBuffer();
+
+ sb.append("\n# Redirections to embbedded page fragments:");
+ sb.append("\n# note: RedirectMatch uses regexp (fuzzy match)");
+
+ final DoiService doiService = doiConfig.getDoiService();
+ final ProjectRepository pr = doiService.getProjectRepository();
+ final DoiRepository dr = doiService.getDoiRepository();
+ final DoiPublicRepository dpr = doiService.getDoiPublicRepository();
+ final DoiStagingRepository dsr = doiService.getDoiStagingRepository();
+
+ final List<Project> projects = pr.findAllByOrderByNameAsc();
+
+ // DOI listing per project:
+ for (Project p : projects) {
+ final String projectName = p.getName();
+
+ final ProjectConfig projectConfig = new ProjectConfig(doiConfig.getPathConfig(), projectName);
+
+ if (projectConfig.getPropertyBoolean(ProjectConfig.CONF_KEY_GENERATE_EMBEDDED)) {
+ // // [public/staging]/PROJECT/embed/DOI_SUFFIX[.html]
+ sb.append("\n# Project ").append(projectName);
+
+ /*
+ RedirectMatch "^/embed/AMMA-CATCH.CE.RainD_Nc([\.-])(.*)" "/staging/AMMA-CATCH/embed/AMMA-CATCH.CE.RainD_Nc$1$2"
+ */
+ final List<Doi> dois = dr.findByProject(projectName);
+
+ logger.info("Dois for project[{}]: {}", projectName, dois.size());
+
+ for (Doi doi : dois) {
+ final String doiSuffix = doi.getIdentifier();
+
+ sb.append("\nRedirectMatch \"^/embed/").append(doiSuffix).append("([\\.-])(.*)\" \"");
+
+ if (doi.getStatus() == Status.PUBLIC) {
+ sb.append('/').append(Paths.DIR_WEB_PUBLIC).append('/');
+ } else {
+ sb.append('/').append(Paths.DIR_WEB_STAGING).append('/');
+ }
+ sb.append(projectName).append('/').append(Paths.DIR_WEB_EMBED).append('/').append(doiSuffix);
+ sb.append("$1$2\"");
+ }
+ sb.append('\n');
+ }
+ }
+ sb.append('\n');
+
+ final String redirectRules = sb.toString();
+
+ FileUtils.writeFile(redirectRules,
+ new File(doiConfig.getPathConfig().getWebRedirectEmbedDir(), FILE_HT_ACCESS));
+
+ logger.debug("generateEmbedUrls:\n{}", redirectRules);
+ }
+
+ private StringBuilder prepareBuffer() {
+ final StringBuilder sb = globalData.buffer;
+ sb.setLength(0);
+
+ // htaccess Header:
+ sb.append("# Generated at ").append(globalData.now);
+ sb.append("\n# Need AllowOverride FileInfo\n");
+ return sb;
+ }
+}
diff --git a/src/main/java/fr/osug/doi/IndexUtil.java b/src/main/java/fr/osug/doi/IndexUtil.java
index 6825c11..736fd3b 100644
--- a/src/main/java/fr/osug/doi/IndexUtil.java
+++ b/src/main/java/fr/osug/doi/IndexUtil.java
@@ -1,5 +1,5 @@
/*******************************************************************************
- * JMMC project ( http://www.jmmc.fr ) - Copyright (C) CNRS.
+ * OSUG-DOI project ( http://doi.osug.fr ) - Copyright (C) CNRS.
******************************************************************************/
package fr.osug.doi;
@@ -12,7 +12,6 @@ import org.slf4j.LoggerFactory;
/**
*
- * @author bourgesl
*/
public class IndexUtil {
diff --git a/src/main/java/fr/osug/doi/PathConfig.java b/src/main/java/fr/osug/doi/PathConfig.java
index 3bfc08f..113610d 100644
--- a/src/main/java/fr/osug/doi/PathConfig.java
+++ b/src/main/java/fr/osug/doi/PathConfig.java
@@ -6,288 +6,87 @@ package fr.osug.doi;
import fr.osug.util.FileUtils;
import java.io.File;
import java.io.IOException;
-import java.io.Reader;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.util.Collections;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.ListIterator;
-import java.util.Map;
-import java.util.Properties;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
*
- * @author bourgesl
*/
-public final class ProjectConfig {
+public final class PathConfig {
- public final static String CONF_PROJECT = "project.properties";
+ private final static Logger logger = LoggerFactory.getLogger(PathConfig.class.getName());
- public final static String CONF_DOI_URL_MAPPING = "doi_url_landing_page.csv";
-
- public final static String CONF_KEY_DATA_ACCESS_URL = "dataAccessUrl";
- public final static String CONF_KEY_GENERATE_EMBEDDED = "generate_embedded";
-
- private final static File[] EMPTY = new File[0];
-
- private final static Logger logger = LoggerFactory.getLogger(ProjectConfig.class.getName());
-
- private final String projectName;
- private final File projectConf;
-
- private final Properties properties;
-
- private final Map<String, String> landingPageUrlMapping;
-
- private final DoiTemplates templates;
- private File[] metadataCsvFiles = EMPTY;
- private File[] inputCsvFiles = EMPTY;
- private File tmpDir;
- private File tmpDoiDir;
- private File stagingDir;
+ /* members */
+ private String webRootPath;
+ private File webRootDir;
private File webStagingDir;
- private File webStagingProjectDir;
- private File webStagingProjectEmbedDir;
- private File webStagingProjectXmlDir;
private File webPublicDir;
- private File webPublicProjectDir;
-
- private boolean saveCSV = false;
-
- public ProjectConfig(final String projectName) throws IOException {
- this.projectName = projectName;
-
- // may fail:
- this.projectConf = FileUtils.getRequiredDirectory(Paths.DIR_CONFIG + projectName).getCanonicalFile();
- logger.info("Project Config: {}", projectConf);
-
- this.properties = loadProperties(projectConf);
-
- this.landingPageUrlMapping = loadUrlMapping(new File(this.projectConf, CONF_DOI_URL_MAPPING));
-
- this.templates = new DoiTemplates(
- new File(this.projectConf, Paths.DIR_PROJECT_TEMPLATES).getAbsolutePath()
- );
-
- initMetadataDir(
- new File(this.projectConf, Paths.DIR_PROJECT_METADATA).getAbsolutePath()
- );
- initInputDir(
- new File(this.projectConf, Paths.DIR_PROJECT_INPUTS).getAbsolutePath()
- );
- initTmpDir(
- new File(Paths.DIR_TMP, projectName).getAbsolutePath()
- );
- initTmpDoiDir(
- new File(tmpDir, Paths.DIR_TMP_DOI).getAbsolutePath()
- );
-
- initStagingDir(
- new File(Paths.DIR_STAGING, projectName).getAbsolutePath()
- );
+ private File webRedirectDir;
+ private File webRedirectEmbedDir;
- // /www/staging directories:
- initWebStagingDir(Paths.DIR_WEB_STAGING);
- initWebStagingProjectDir(
- new File(Paths.DIR_WEB_STAGING, projectName).getAbsolutePath()
- );
- initWebStagingProjectEmbedDir(
- new File(webStagingProjectDir, Paths.DIR_WEB_EMBED).getAbsolutePath()
- );
- initWebStagingProjectXmlDir(
- new File(webStagingProjectDir, Paths.DIR_WEB_XML).getAbsolutePath()
- );
- // /www/public directories:
- initWebPublicDir(Paths.DIR_WEB_PUBLIC);
- initWebPublicProjectDir(
- new File(Paths.DIR_WEB_PUBLIC, projectName).getAbsolutePath()
- );
+ public PathConfig(final String webRootDir) throws IOException {
+ initWebRoot(webRootDir);
}
- public void initMetadataDir(final String dirPath) throws IOException {
- this.metadataCsvFiles = CsvUtil.findCSVFiles(dirPath);
+ public void initWebRoot(final String webRootPath) throws IOException {
+ logger.info("web root: {}", webRootPath);
+
+ this.webRootPath = webRootPath;
+ // /www/ directories:
+ initWebRootDir(webRootPath);
+ // /www/staging directory:
+ initWebStagingDir(webRootPath + Paths.DIR_WEB_STAGING);
+ // /www/public directory:
+ initWebPublicDir(webRootPath + Paths.DIR_WEB_PUBLIC);
+ // Redirect directories:
+ // /www/r directory:
+ initWebRedirectDir(webRootPath + Paths.DIR_WEB_REDIRECT);
+ // /www/embed directory:
+ initWebRedirectEmbedDir(webRootPath + Paths.DIR_WEB_EMBED);
}
- // Used by junit test
- public void initInputDir(final String dirPath) throws IOException {
- this.inputCsvFiles = CsvUtil.findCSVFiles(dirPath);
+ private void initWebRootDir(final String path) throws IOException {
+ this.webRootDir = FileUtils.createDirectories(path);
}
- public void initTmpDir(final String path) throws IOException {
- this.tmpDir = FileUtils.createDirectories(path);
- }
-
- public void initTmpDoiDir(final String path) throws IOException {
- this.tmpDoiDir = FileUtils.createDirectories(path);
- }
-
- public void initStagingDir(final String path) throws IOException {
- this.stagingDir = FileUtils.createDirectories(path);
- }
-
- public void initWebStagingDir(final String path) throws IOException {
+ private void initWebStagingDir(final String path) throws IOException {
this.webStagingDir = FileUtils.createDirectories(path);
}
- public void initWebStagingProjectDir(final String path) throws IOException {
- this.webStagingProjectDir = FileUtils.createDirectories(path);
- }
-
- public void initWebStagingProjectEmbedDir(final String path) throws IOException {
- this.webStagingProjectEmbedDir = FileUtils.createDirectories(path);
- }
-
- public void initWebStagingProjectXmlDir(final String path) throws IOException {
- this.webStagingProjectXmlDir = FileUtils.createDirectories(path);
- }
-
- public void initWebPublicDir(final String path) throws IOException {
+ private void initWebPublicDir(final String path) throws IOException {
this.webPublicDir = FileUtils.createDirectories(path);
}
- public void initWebPublicProjectDir(final String path) throws IOException {
- this.webPublicProjectDir = FileUtils.createDirectories(path);
+ private void initWebRedirectDir(final String path) throws IOException {
+ this.webRedirectDir = FileUtils.createDirectories(path);
}
- public String getProjectName() {
- return projectName;
+ private void initWebRedirectEmbedDir(final String path) throws IOException {
+ this.webRedirectEmbedDir = FileUtils.createDirectories(path);
}
- public File getProjectConf() {
- return projectConf;
+ public String getWebRootPath() {
+ return webRootPath;
}
- public String getProperty(final String key) {
- final String val = properties.getProperty(key);
- if (val != null) {
- return val.trim();
- }
- return null;
- }
-
- public boolean getPropertyBoolean(final String key) {
- return Boolean.valueOf(properties.getProperty(key, "false"));
- }
-
- public String getLandingPageUrl(final String doiSuffix) {
- return landingPageUrlMapping.get(doiSuffix);
- }
-
- public DoiTemplates getTemplates() {
- return templates;
- }
-
- public File[] getMetadataCsvFiles() {
- return metadataCsvFiles;
- }
-
- public File[] getInputCsvFiles() {
- return inputCsvFiles;
- }
-
- public File getTmpDir() {
- return tmpDir;
- }
-
- public File getTmpDoiDir() {
- return tmpDoiDir;
- }
-
- public File getStagingDir() {
- return stagingDir;
+ public File getWebRootDir() {
+ return webRootDir;
}
public File getWebStagingDir() {
return webStagingDir;
}
- public File getWebStagingProjectDir() {
- return webStagingProjectDir;
- }
-
- public File getWebStagingProjectEmbedDir() {
- return webStagingProjectEmbedDir;
- }
-
- public File getWebStagingProjectXmlDir() {
- return webStagingProjectXmlDir;
- }
-
public File getWebPublicDir() {
return webPublicDir;
}
- public File getWebPublicProjectDir() {
- return webPublicProjectDir;
- }
-
- public boolean isSaveCSV() {
- return saveCSV;
- }
-
- public void setSaveCSV(boolean saveCSV) {
- this.saveCSV = saveCSV;
+ public File getWebRedirectDir() {
+ return webRedirectDir;
}
- private static Properties loadProperties(final File projectConf) {
- final File propFile = new File(projectConf, CONF_PROJECT);
- logger.info("Loading {}", propFile);
-
- final Properties props = new Properties();
- Reader r = null;
- try {
- r = FileUtils.reader(propFile);
- props.load(r);
- } catch (IOException ioe) {
- throw new RuntimeException("loadProperties failed:" + propFile.getAbsolutePath(), ioe);
- } finally {
- FileUtils.closeFile(r);
- }
- logger.info("properties: {}", props);
- return props;
+ public File getWebRedirectEmbedDir() {
+ return webRedirectEmbedDir;
}
- private static Map<String, String> loadUrlMapping(final File csvFile) throws IOException {
- final Map<String, String> urlMapping;
-
- if (csvFile.exists()) {
- final CsvData data = CsvUtil.read(csvFile);
- logger.debug("CSV data: {}", data);
-
- final List<String[]> rows = data.getRows();
-
- urlMapping = new LinkedHashMap<String, String>(rows.size());
-
- // Get mappings [DOI|URL]
- for (ListIterator<String[]> it = rows.listIterator(); it.hasNext();) {
- final String[] cols = it.next();
- if (cols == null || cols.length != 2) {
- // remove line:
- it.remove();
- continue;
- }
- String key = cols[0].trim();
- if (key.isEmpty() || key.charAt(0) == Const.COMMENT) {
- continue;
- }
- String url = cols[1].trim();
- if (url.isEmpty() || !url.startsWith("http")) {
- continue;
- }
- try {
- // should detect repeated keys or values ?
- urlMapping.put(key, new URL(url).toString());
- } catch (MalformedURLException mue) {
- logger.info("invalid URL: {}", url);
- }
- }
- } else {
- urlMapping = Collections.emptyMap();
- }
- logger.info("urlMapping: {}", urlMapping);
- return urlMapping;
- }
}
diff --git a/src/main/java/fr/osug/doi/Paths.java b/src/main/java/fr/osug/doi/Paths.java
index 78f42c6..0ecc530 100644
--- a/src/main/java/fr/osug/doi/Paths.java
+++ b/src/main/java/fr/osug/doi/Paths.java
@@ -11,7 +11,6 @@ import org.slf4j.LoggerFactory;
/**
*
- * @author bourgesl
*/
public final class Paths {
@@ -45,15 +44,21 @@ public final class Paths {
/* [base]/tmp/[PROJECT]/doi/ */
public final static String DIR_TMP_DOI = "doi/";
- /* [base]/staging directory (DOI) */
- public final static String DIR_STAGING = Paths.DIR_BASE + "staging/";
+ /* [base]/data directory (DOI) */
+ public final static String DIR_DATA = Paths.DIR_BASE + "data/";
+ /* [base]/data/staging directory (DOI) */
+ public final static String DIR_STAGING = DIR_DATA + "staging/";
+ /* [base]/data/public directory (DOI) */
+ public final static String DIR_PUBLIC = DIR_DATA + "public/";
- /* [base]/www directory (web) */
+ /* [base]/www directory = web root */
public final static String DIR_WEB = Paths.DIR_BASE + "www/";
- /* [base]/www/staging/ */
- public final static String DIR_WEB_STAGING = DIR_WEB + "staging/";
- /* [base]/www/public/ */
- public final static String DIR_WEB_PUBLIC = DIR_WEB + "public/";
+ /* web sub-directory /r */
+ public final static String DIR_WEB_REDIRECT = "r";
+ /* web sub-directory /staging */
+ public final static String DIR_WEB_STAGING = "staging";
+ /* web sub-directory /public */
+ public final static String DIR_WEB_PUBLIC = "public";
/* web sub-directory /embed */
public final static String DIR_WEB_EMBED = "embed";
/* web sub-directory /xml */
diff --git a/src/main/java/fr/osug/doi/PipelineCommonData.java b/src/main/java/fr/osug/doi/PipelineCommonData.java
new file mode 100644
index 0000000..1268939
--- /dev/null
+++ b/src/main/java/fr/osug/doi/PipelineCommonData.java
@@ -0,0 +1,20 @@
+/*******************************************************************************
+ * OSUG-DOI project ( http://doi.osug.fr ) - Copyright (C) CNRS.
+ ******************************************************************************/
+package fr.osug.doi;
+
+import java.util.Date;
+
+/**
+ *
+ */
+public class PipelineCommonData {
+
+ public static final int BUFFER_SIZE = 4 * 1024;
+
+ /* members */
+ final Date now = new Date();
+ /** temporary StringBuilder to hold xml document */
+ final StringBuilder buffer = new StringBuilder(BUFFER_SIZE);
+
+}
diff --git a/src/main/java/fr/osug/doi/PipelineConfig.java b/src/main/java/fr/osug/doi/PipelineConfig.java
new file mode 100644
index 0000000..288fa70
--- /dev/null
+++ b/src/main/java/fr/osug/doi/PipelineConfig.java
@@ -0,0 +1,12 @@
+/*******************************************************************************
+ * JMMC project ( http://www.jmmc.fr ) - Copyright (C) CNRS.
+ ******************************************************************************/
+package fr.osug.doi;
+
+/**
+ *
+ * @author bourgesl
+ */
+public class PipelineConfig {
+
+}
diff --git a/src/main/java/fr/osug/doi/ProcessPipeline.java b/src/main/java/fr/osug/doi/ProcessPipeline.java
index bf3fbb1..22e9d5a 100644
--- a/src/main/java/fr/osug/doi/ProcessPipeline.java
+++ b/src/main/java/fr/osug/doi/ProcessPipeline.java
@@ -4,12 +4,9 @@
package fr.osug.doi;
import fr.osug.doi.domain.Doi;
-import fr.osug.doi.domain.DoiStaging;
-import fr.osug.doi.domain.Project;
-import fr.osug.doi.domain.IndexEntry;
import fr.osug.doi.domain.NameEntry;
-import fr.osug.doi.repository.DoiStagingRepository;
-import fr.osug.doi.repository.ProjectRepository;
+import fr.osug.doi.domain.Status;
+import fr.osug.doi.service.DataciteClient;
import fr.osug.doi.service.DoiService;
import fr.osug.doi.validation.ValidationUtil;
import fr.osug.util.FileUtils;
@@ -22,358 +19,118 @@ import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Date;
-import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.ListIterator;
-import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import javax.xml.transform.stream.StreamResult;
import javax.xml.transform.stream.StreamSource;
-import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.springframework.transaction.annotation.Transactional;
/**
*
- * @author bourgesl
*/
-public final class ProcessPipeline {
+public class ProcessPipeline {
private final static Logger logger = LoggerFactory.getLogger(ProcessPipeline.class.getName());
- public final static String HTML_EXT = ".html";
- public final static String HTML_INDEX = "index" + HTML_EXT;
- public final static String HTML_REPORT = "report" + HTML_EXT;
- public final static String HTML_ERROR = "error" + HTML_EXT;
-
public final static String XSD_DOI_SCHEMA = "file://" + Paths.DIR_XSD_SCHEMA + "/metadata.xsd";
private final static String XSLT_XML_TO_DOI = "xml2doi.xsl";
- private final static String XSLT_DOI_TO_LANDING_PAGE = "doi2landing.xsl";
-
- private final static String XSLT_PROJECT_INDEX = "project_index.xsl";
- private final static String XSLT_PROJECT_REPORT = "project_report.xsl";
-
- private final static String CONFIG_ACCESS_INSTRUCTIONS = "access_instruction.html";
-
- // xslt parameters (common):
- private final static String XSLT_PARAM_DATE = "date";
- private final static String XSLT_PARAM_WEB_ROOT = "www_root";
- // xslt parameters (doi2landing):
- private final static String XSLT_PARAM_DATA_ACCESS_HTML = "data_access_html";
- private final static String XSLT_PARAM_DATA_ACCESS_URL = "data_access_url";
- private final static String XSLT_PARAM_EMBEDDED = "embedded";
- private final static String XSLT_PARAM_EMBEDDED_FULL = "full";
- private final static String XSLT_PARAM_EMBEDDED_HEADER = "header";
- private final static String XSLT_PARAM_EMBEDDED_META = "meta";
- private final static String XSLT_PARAM_LANDING_PAGE_URL = "landing_page_url";
- private final static String XSLT_PARAM_METADATA_FILE = "metadataFile";
-
- // xslt parameters (index_project):
- private final static String XSLT_PARAM_PARENT = "parent";
- private final static String XSLT_PARAM_CURRENT = "current";
+ private final static String FILE_NAMES_CSV = "names" + Const.FILE_EXT_CSV;
// members
- private Date _now = null;
private final DOIConfig doiConfig;
private final ProjectConfig projectConfig;
// temporary data
- private final PipelineGlobalData globalData = new PipelineGlobalData();
+ private final ProcessPipelineData globalData = new ProcessPipelineData();
+ /** child pipeline */
+ private final GeneratePipeline pipe;
public ProcessPipeline(final DOIConfig doiConfig,
final ProjectConfig projectConfig) throws IOException {
this.doiConfig = doiConfig;
this.projectConfig = projectConfig;
+
+ // preload all needed data:
+ projectConfig.prepareProcess();
+
+ // child pipeline:
+ this.pipe = new GeneratePipeline(doiConfig, true, false, globalData);
}
public void process() throws IOException {
+ File[] files;
// 1 - Process complete datasets (csv):
- for (File inputCsvFile : projectConfig.getMetadataCsvFiles()) {
+ files = CsvUtil.findCSVFiles(projectConfig.getMetadataDir());
+ for (File inputCsvFile : files) {
processCsvFile(inputCsvFile, false);
}
// 2 - Process partial datasets (csv to be merged with templates):
- for (File inputCsvFile : projectConfig.getInputCsvFiles()) {
+ files = CsvUtil.findCSVFiles(projectConfig.getInputDir());
+ for (File inputCsvFile : files) {
processCsvFile(inputCsvFile, true);
}
// TODO: 3 - process complete datasets as XML files (full metadata)
+ // use case ?
// global validation of references
validateRefs();
- // 4 - Landing pages
- generateLandingPages();
+ // 4 - Landing pages:
+ pipe.generateLandingPages(projectConfig);
+
+ // test publication datacite ?
+ publishDatacite();
- // TODO: test publication datacite ?
// Update database:
updateDatabase();
// Update index pages:
- generateIndexPagesStaging();
+ pipe.generateIndexPages();
}
- private void generateLandingPages() throws IOException {
-
- final File webStaging = projectConfig.getWebStagingProjectDir();
- logger.info("Generating landing pages into {}", webStaging);
-
- final File webStagingEmbed = projectConfig.getWebStagingProjectEmbedDir();
- final File webStagingXml = projectConfig.getWebStagingProjectXmlDir();
-
- final File dataAccessFileLocation = new File(projectConfig.getProjectConf(), CONFIG_ACCESS_INSTRUCTIONS);
- final File dataAccessFile = FileUtils.getFile(dataAccessFileLocation);
- if (dataAccessFile == null) {
- logger.warn("Missing dataAccessFile: {}", dataAccessFileLocation);
+ private void publishDatacite() throws IOException {
+ if (!doiConfig.isDataciteClientEnabled()) {
+ logger.warn("Datacite client disabled.");
+ return;
}
- // TODO: use mapping file [DOI:dataAccessUrl] like url mapping:
- final String defaultDataAccessUrl = projectConfig.getProperty(ProjectConfig.CONF_KEY_DATA_ACCESS_URL);
+ final DataciteClient dc = doiConfig.getDataciteClient();
- final Map<String, Object> xslParameters = new HashMap<String, Object>(8);
- xslParameters.put(XSLT_PARAM_DATE, now().toString());
- if (dataAccessFile != null) {
- xslParameters.put(XSLT_PARAM_DATA_ACCESS_HTML, dataAccessFile.toURI().toString());
- }
- if (defaultDataAccessUrl != null) {
- xslParameters.put(XSLT_PARAM_DATA_ACCESS_URL, defaultDataAccessUrl);
- }
+ logger.info("Datacite DOI list:\n{}", dc.getDois());
- for (PipelineDoiData doiData : globalData.getDoiDatas().values()) {
+ for (ProcessPipelineDoiData doiData : globalData.getDoiDatas().values()) {
final String doiSuffix = doiData.getDoiSuffix();
- logger.info("processing {}", doiSuffix);
- // Get external Landing page URL:
- final String landingPageUrl = projectConfig.getLandingPageUrl(doiSuffix);
- if (landingPageUrl != null) {
- logger.info("landingPageUrl: {}", landingPageUrl);
- xslParameters.put(XSLT_PARAM_LANDING_PAGE_URL, landingPageUrl);
+ final String doi = Const.DOI_PREFIX_TEST + '/' + doiSuffix;
+ // short url:
+ final String url = doiConfig.getDomain() + '/' + doiSuffix;
- doiData.setLandingPageUrl(landingPageUrl);
- } else {
- xslParameters.remove(XSLT_PARAM_LANDING_PAGE_URL);
- }
-
- // add warnings:
- if (dataAccessFile == null) {
- doiData.addError("Missing data access instructions (" + CONFIG_ACCESS_INSTRUCTIONS + ")");
- }
- if (StringUtils.isEmpty(defaultDataAccessUrl)) {
- doiData.addError("Missing data access url");
- }
-
- final File stagingFileDOI = doiData.getStagingFileDOI();
+ logger.info("publish DOI [{}]", doi);
- if (stagingFileDOI == null) {
- logger.info("Ignoring {} (no xml)", doiSuffix);
- } else {
- final String filenameNoExt = FileUtils.getFileNameWithoutExtension(stagingFileDOI);
-
- // main landing page:
- xslParameters.remove(XSLT_PARAM_EMBEDDED);
- xslParameters.put(XSLT_PARAM_WEB_ROOT, "../../");
- File output = new File(webStaging, filenameNoExt + HTML_EXT);
- generateLandingPage(xslParameters, stagingFileDOI, output);
-
- if (projectConfig.getPropertyBoolean(ProjectConfig.CONF_KEY_GENERATE_EMBEDDED)) {
- // embedded mode:
- xslParameters.put(XSLT_PARAM_WEB_ROOT, "../../../");
-
- // embedded mode - full:
- output = new File(webStagingEmbed, filenameNoExt + HTML_EXT);
- xslParameters.put(XSLT_PARAM_EMBEDDED, XSLT_PARAM_EMBEDDED_FULL);
- generateLandingPage(xslParameters, stagingFileDOI, output);
-
- // embedded mode - header:
- output = new File(webStagingEmbed, filenameNoExt + "-header" + HTML_EXT);
- xslParameters.put(XSLT_PARAM_EMBEDDED, XSLT_PARAM_EMBEDDED_HEADER);
- generateLandingPage(xslParameters, stagingFileDOI, output);
-
- // embedded mode - meta:
- output = new File(webStagingEmbed, filenameNoExt + "-meta" + HTML_EXT);
- xslParameters.put(XSLT_PARAM_EMBEDDED, XSLT_PARAM_EMBEDDED_META);
- generateLandingPage(xslParameters, stagingFileDOI, output);
- }
+ // Push DOI in staging phase:
+ final String metadata = FileUtils.readFile(doiData.getStagingFileDOI());
- // Xml DOI:
- output = new File(webStagingXml, stagingFileDOI.getName());
- FileUtils.copy(stagingFileDOI, output);
- }
+ dc.publishDoi(doi, url, metadata);
}
}
- private void generateLandingPage(final Map<String, Object> xslParameters,
- final File stagingFileDOI, final File output) {
-
- logger.info("generateLandingPage: {}", output);
-
- xslParameters.put(XSLT_PARAM_METADATA_FILE, stagingFileDOI.getName());
- logger.debug("xslParameters: {}", xslParameters);
-
- // use XSLT to generate landing page from the doi xml file:
- doiConfig.getXmlFactory().transform(
- new StreamSource(stagingFileDOI),
- Paths.DIR_XSL + XSLT_DOI_TO_LANDING_PAGE,
- xslParameters, true,
- new StreamResult(output)
- );
- }
-
- @Transactional
- /* public needed by Transactional */
- public void updateDatabase() {
+ private void updateDatabase() {
+ logger.info("updateDatabase ...");
final String projectName = projectConfig.getProjectName();
+ final Date now = globalData.now;
- final Date now = now();
-
- // Get or create the project:
- final DoiService doiService = doiConfig.getDoiService();
- final Project project = doiService.getOrCreateProject(projectName, now);
-
- // Update Dois:
- for (PipelineDoiData doiData : globalData.getDoiDatas().values()) {
- // Update DOI in staging phase:
- doiService.createOrUpdateStagingDoi(project, doiData.getDoiSuffix(),
- doiData.getTitle(), doiData.getLandingPageUrl(),
- doiData.isValid(), doiData.getMetadataMd5(),
- doiData.messagesToString(), now);
- }
-
- // Update Project:
- doiService.updateProjectDate(projectName, now);
- }
-
- @Transactional
- /* public needed by Transactional */
- public void generateIndexPagesStaging() {
- final File webStaging = projectConfig.getWebStagingDir();
-
- final DoiService doiService = doiConfig.getDoiService();
- final ProjectRepository pr = doiService.getProjectRepository();
- final DoiStagingRepository dsr = doiService.getDoiStagingRepository();
-
- final List<Project> projects = pr.findAllByOrderByNameAsc();
-
- // Project listing:
- List<IndexEntry> entries = new ArrayList<IndexEntry>();
-
- for (Project p : projects) {
- final String projectName = p.getName();
- final Long countDoi = dsr.countByProject(projectName);
- final IndexEntry ie = new IndexEntry(projectName, p.getDescription(), p.getUpdateDate(), countDoi);
-
- logger.debug("IndexEntry: {}", ie);
- entries.add(ie);
- }
- // serialize entries and transform in the /staging/index.html
- IndexUtil.write(entries, globalData.buffer);
-
- String xmlDoc = globalData.buffer.toString();
- logger.debug("xmlDoc(project index): \n{}", xmlDoc);
-
- final Map<String, Object> xslParameters = new HashMap<String, Object>(8);
- xslParameters.put(XSLT_PARAM_DATE, now().toString());
- xslParameters.put(XSLT_PARAM_CURRENT, "staging");
- xslParameters.put(XSLT_PARAM_WEB_ROOT, "../");
- File output = new File(webStaging, HTML_INDEX);
-
- // use XSLT to generate the index page:
- doiConfig.getXmlFactory().transform(
- new StreamSource(new StringReader(xmlDoc)),
- Paths.DIR_XSL_ROOT + XSLT_PROJECT_INDEX,
- xslParameters, true,
- new StreamResult(output)
- );
-
- // DOI listing per project:
- for (Project p : projects) {
- final String projectName = p.getName();
- final List<DoiStaging> dsList = dsr.findByProject(projectName);
-
- logger.debug("DoiStaging list for project[{}]: {}", projectName, dsList);
-
- // index.html:
- entries.clear();
-
- for (DoiStaging ds : dsList) {
- logger.debug("DoiStaging: {}", ds);
-
- final Doi doi = ds.getDoi();
-
- final IndexEntry ie = new IndexEntry(doi.getIdentifier(), doi.getDescription(), ds.getUpdateDate());
+ // To encapsulate all write operations in a single transaction (all or nothing):
+ doiConfig.getDoiService().importDoiStagingCollection(projectName, now, globalData.getDoiDatas().values());
- logger.debug("IndexEntry: {}", ie);
- entries.add(ie);
- }
- // serialize entries and transform in the /staging/PROJECT/index.html
- IndexUtil.write(entries, globalData.buffer);
-
- xmlDoc = globalData.buffer.toString();
- logger.debug("xmlDoc(doi index): \n{}", xmlDoc);
-
- xslParameters.put(XSLT_PARAM_PARENT, "staging");
- xslParameters.put(XSLT_PARAM_CURRENT, projectName);
- xslParameters.put(XSLT_PARAM_WEB_ROOT, "../../");
- // suppose child directory is the project folder:
- output = new File(new File(webStaging, projectName), HTML_INDEX);
-
- // use XSLT to generate the index page:
- doiConfig.getXmlFactory().transform(
- new StreamSource(new StringReader(xmlDoc)),
- Paths.DIR_XSL_ROOT + XSLT_PROJECT_INDEX,
- xslParameters, true,
- new StreamResult(output)
- );
-
- // serialize doi staging list and transform in the /staging/PROJECT/report.html
- IndexUtil.writeReport(dsList, globalData.buffer);
-
- xmlDoc = globalData.buffer.toString();
- logger.debug("xmlDoc(doi report): \n{}", xmlDoc);
-
- xslParameters.put(XSLT_PARAM_PARENT, "staging");
- xslParameters.put(XSLT_PARAM_CURRENT, projectName);
- xslParameters.put(XSLT_PARAM_WEB_ROOT, "../../");
- // suppose child directory is the project folder:
- output = new File(new File(webStaging, projectName), HTML_REPORT);
-
- // use XSLT to generate the report page:
- doiConfig.getXmlFactory().transform(
- new StreamSource(new StringReader(xmlDoc)),
- Paths.DIR_XSL_ROOT + XSLT_PROJECT_REPORT,
- xslParameters, true,
- new StreamResult(output)
- );
-
- // serialize doi staging error list and transform in the /staging/PROJECT/error.html
- final List<DoiStaging> dsErrorList = dsr.findErrorByProject(projectName);
- logger.debug("DoiStaging error list for project[{}]: {}", projectName, dsErrorList);
-
- IndexUtil.writeReport(dsErrorList, globalData.buffer);
-
- xmlDoc = globalData.buffer.toString();
- logger.debug("xmlDoc(error report): \n{}", xmlDoc);
-
- xslParameters.put(XSLT_PARAM_PARENT, "staging");
- xslParameters.put(XSLT_PARAM_CURRENT, projectName);
- xslParameters.put(XSLT_PARAM_WEB_ROOT, "../../");
- // suppose child directory is the project folder:
- output = new File(new File(webStaging, projectName), HTML_ERROR);
-
- // use XSLT to generate the report page:
- doiConfig.getXmlFactory().transform(
- new StreamSource(new StringReader(xmlDoc)),
- Paths.DIR_XSL_ROOT + XSLT_PROJECT_REPORT,
- xslParameters, true,
- new StreamResult(output)
- );
- }
- logger.info("generateIndexPagesStaging: done.");
+ logger.info("updateDatabase: done");
}
private void validateRefs() throws IOException {
@@ -388,8 +145,9 @@ public final class ProcessPipeline {
final Set<String> refs = globalData.getDoiRefs(doiId);
if (refs != null) {
for (String refId : refs) {
+ final boolean isPublic = refId.startsWith(publicPrefix);
// Only check DOI references with Prefix Test or Public:
- if (ValidationUtil.isTestPrefix(refId) || refId.startsWith(publicPrefix)) {
+ if (ValidationUtil.isTestPrefix(refId) || isPublic) {
logger.debug("validateRefs: check ref[{}]", refId);
boolean found = false;
@@ -400,12 +158,19 @@ public final class ProcessPipeline {
} else {
// Then in database (previous data):
final String doiSuffix = DoiCsvData.getDoiSuffix(refId);
- if (doiService.getDoi(doiSuffix) != null) {
- found = true;
+
+ final Doi doi = doiService.getDoi(doiSuffix);
+ if (doi != null) {
+ if (isPublic && doi.getStatus() != Status.PUBLIC) {
+ logger.warn("validateRefs: DOI[{}] is not public yet (use test prefix instead)", refId);
+ found = false;
+ } else {
+ found = true;
+ }
}
}
if (!found) {
- final PipelineDoiData doiData = globalData.getDoiData(doiId);
+ final ProcessPipelineDoiData doiData = globalData.getDoiData(doiId);
if (doiData != null) {
doiData.addError("Invalid " + Const.KEY_REL_ID_START + " '" + refId + "' found.");
}
@@ -462,7 +227,7 @@ public final class ProcessPipeline {
}
// Save file as CSV:
- final File fileCSV = new File(projectConfig.getTmpDir(), "names" + Const.FILE_EXT_CSV);
+ final File fileCSV = new File(projectConfig.getTmpDir(), FILE_NAMES_CSV);
CsvUtil.write(data, fileCSV);
}
@@ -486,7 +251,7 @@ public final class ProcessPipeline {
final String doiSuffix = data.getDoiSuffix();
logger.info("process: dataset[{}].", doiSuffix);
- final PipelineDoiData doiData = globalData.newDoiData(doiId, doiSuffix, data.getTitle());
+ final ProcessPipelineDoiData doiData = globalData.newDoiData(doiId, doiSuffix, data.getTitle());
// ignore duplicates
if (doiData != null) {
@@ -508,7 +273,7 @@ public final class ProcessPipeline {
}
}
- private void mergeCSV(final DoiCsvData data, final PipelineDoiData doiData) {
+ private void mergeCSV(final DoiCsvData data, final ProcessPipelineDoiData doiData) throws IOException {
final DoiTemplates templates = projectConfig.getTemplates();
@@ -526,59 +291,82 @@ public final class ProcessPipeline {
final String doiId = data.getDoiId(); // not null
- /*
- ### 3/ Traitement spécifiques: fusionner avec template pays
+ if (projectConfig.getPropertyBoolean(ProjectConfig.CONF_KEY_USE_TEMPLATE_COUNTRY)) {
+ /*
+ ### 3/ Traitement spécifiques: fusionner avec template pays
- Fusionner avec template pays (template_benin.csv):
+ Fusionner avec template pays (template_benin.csv):
- Trouver le template du pays qui contient geoLocationPlace;Benin
- */
- final String geoLocationPlace = data.getFirstGeoLocationPlace();
- if (geoLocationPlace == null) {
- doiData.addWarning("Missing ''" + Const.KEY_GEO_LOCATION_PLACE + "'' in dataset '" + doiId + "'");
- } else {
- t = templates.getByGeoLocationPlace(geoLocationPlace);
- if (t == null) {
- doiData.addError("Missing country template for '" + geoLocationPlace + "'");
+ Trouver le template du pays qui contient geoLocationPlace;Benin
+ */
+ final String geoLocationPlace = data.getFirstGeoLocationPlace();
+ if (geoLocationPlace == null) {
+ doiData.addWarning("Missing ''" + Const.KEY_GEO_LOCATION_PLACE + "'' in dataset '" + doiId + "'");
} else {
- logger.info("Using country template '{}'", geoLocationPlace);
- data.mergeFrom(t);
+ t = templates.getByGeoLocationPlace(geoLocationPlace);
+ if (t == null) {
+ doiData.addError("Missing country template for '" + geoLocationPlace + "'");
+ } else {
+ logger.info("Using country template '{}'", geoLocationPlace);
+ data.mergeFrom(t);
+ }
}
}
- /*
- ### 4/ Traitement spécifiques: fusionner avec template jeu
+ if (projectConfig.getPropertyBoolean(ProjectConfig.CONF_KEY_USE_TEMPLATE_DOI)) {
+ /*
+ ### 4/ Traitement spécifiques: fusionner avec template jeu
- Trouver le nom du jeu dans le XML:
- <identifier identifierType="DOI">10.5072/AMMA-CATCH.CL.Run_O</identifier>
+ Trouver le nom du jeu dans le XML:
+ <identifier identifierType="DOI">10.5072/AMMA-CATCH.CL.Run_O</identifier>
- Trouver le template du jeu qui contient identifier:DOI;10.5072/AMMA-CATCH.CL.Run_O
- */
- t = templates.getByIdentifier(doiId);
- if (t == null) {
- doiData.addWarning("Missing dataset template for id '" + doiId + "'");
- } else {
- logger.info("Using dataset template for id '{}'", doiId);
- data.mergeFrom(t);
+ Trouver le template du jeu qui contient identifier:DOI;10.5072/AMMA-CATCH.CL.Run_O
+ */
+ t = templates.getByIdentifier(doiId);
+ if (t == null) {
+ doiData.addWarning("Missing dataset template for id '" + doiId + "'");
+ } else {
+ logger.info("Using dataset template for id '{}'", doiId);
+ data.mergeFrom(t);
+ }
}
}
- private void validateData(final DoiCsvData data, final PipelineDoiData doiData) {
+ private void validateData(final DoiCsvData data, final ProcessPipelineDoiData doiData) throws IOException {
logger.debug("validateData [{}]", doiData.getDoiId());
- // TODO: check title ?
- // check all values:
+ // check title
+ if (data.getTitle().isEmpty()) {
+ doiData.addWarning("Empty title.");
+ }
+
final List<String[]> rows = data.getRows();
+
+ // check all values:
for (ListIterator<String[]> it = rows.listIterator(); it.hasNext();) {
- final String[] cols = it.next();
+ String[] cols = it.next();
if (cols != null && cols.length >= 2) {
if (cols[1].toLowerCase().contains("todo")) {
doiData.addWarning("TODO detected for key [" + cols[0] + ']');
}
- // Collect name references:
+ // Collect name references and override specific name entries:
final String key = cols[0];
if (key.startsWith(Const.KEY_CREATOR_NAME) || key.startsWith(Const.KEY_CONTRIBUTOR_NAME)) {
+ final String name = cols[1];
+ final String simpleName = ValidationUtil.simplifyName(name);
+
+ // Check in override names:
+ final NameEntry override = projectConfig.getOverrideNameEntry(simpleName);
+
+ if (override != null) {
+ logger.debug("override: {}", override);
+ cols = NameEntry.toCsv(override);
+ cols[0] = key;
+
+ it.set(cols);
+ }
+
globalData.addNameRef(cols);
}
}
@@ -592,7 +380,7 @@ public final class ProcessPipeline {
globalData.addRefs(doiData.getDoiId(), refs);
}
- private void saveData(final DoiCsvData data, final PipelineDoiData doiData) throws IOException {
+ private void saveData(final DoiCsvData data, final ProcessPipelineDoiData doiData) throws IOException {
// Sort keys:
data.sort();
@@ -634,7 +422,7 @@ public final class ProcessPipeline {
doiData.addError("No Metadata XML output");
} else {
final File fileDOI = new File(tmpDoiDir, baseName + Const.FILE_EXT_XML);
- DoiCsvToXml.writeXml(doiDoc, fileDOI);
+ FileUtils.writeFile(doiDoc, fileDOI);
// compute MD5 on metadata:
final String xmlMd5 = FileUtils.MD5(new FileInputStream(fileDOI));
@@ -678,14 +466,7 @@ public final class ProcessPipeline {
if (forceSaveXML || doiConfig.isDebug()) {
final File fileXML = new File(tmpDoiDir, baseName + "-xml" + Const.FILE_EXT_XML);
- DoiCsvToXml.writeXml(xmlDoc, fileXML);
- }
- }
-
- private Date now() {
- if (_now == null) {
- _now = new Date();
+ FileUtils.writeFile(xmlDoc, fileXML);
}
- return _now;
}
}
diff --git a/src/main/java/fr/osug/doi/ProcessPipelineData.java b/src/main/java/fr/osug/doi/ProcessPipelineData.java
index 970e7c8..e012075 100644
--- a/src/main/java/fr/osug/doi/ProcessPipelineData.java
+++ b/src/main/java/fr/osug/doi/ProcessPipelineData.java
@@ -1,12 +1,11 @@
/*******************************************************************************
- * JMMC project ( http://www.jmmc.fr ) - Copyright (C) CNRS.
+ * OSUG-DOI project ( http://doi.osug.fr ) - Copyright (C) CNRS.
******************************************************************************/
package fr.osug.doi;
import fr.osug.doi.domain.NameEntry;
import fr.osug.util.StringBuilderWriter;
import java.util.Collections;
-import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Set;
@@ -14,45 +13,40 @@ import java.util.TreeSet;
/**
*
- * @author bourgesl
*/
-public final class PipelineGlobalData {
+public final class ProcessPipelineData extends PipelineCommonData {
private final static Set<String> NO_REFS = Collections.emptySet();
- private static final int BUFFER_SIZE = 4 * 1024;
-
/* members */
- /** temporary StringBuilder to hold xml document */
- final StringBuilder buffer = new StringBuilder(BUFFER_SIZE);
/** temporary StringBuilderWriter to hold doi document */
final StringBuilderWriter doiDoc = new StringBuilderWriter(BUFFER_SIZE);
// References:
private final Map<String, Set<String>> doiRefs = new LinkedHashMap<String, Set<String>>();
/* doi data (insertion order) */
- private final Map<String, PipelineDoiData> datas = new LinkedHashMap<String, PipelineDoiData>();
+ private final Map<String, ProcessPipelineDoiData> datas = new LinkedHashMap<String, ProcessPipelineDoiData>();
// Name references:
private final TreeSet<NameEntry> nameRefs = new TreeSet<NameEntry>(NameEntry.COMPARATOR);
- public Map<String, PipelineDoiData> getDoiDatas() {
+ public Map<String, ProcessPipelineDoiData> getDoiDatas() {
return datas;
}
- PipelineDoiData newDoiData(final String doiId, final String doiSuffix, final String title) {
- PipelineDoiData doiData = getDoiData(doiId);
+ ProcessPipelineDoiData newDoiData(final String doiId, final String doiSuffix, final String title) {
+ ProcessPipelineDoiData doiData = getDoiData(doiId);
// ensure unicity:
if (doiData != null) {
doiData.addError("Duplicate detected for DOI: " + doiSuffix);
// ignore later
doiData = null;
} else {
- doiData = new PipelineDoiData(doiId, doiSuffix, title);
+ doiData = new ProcessPipelineDoiData(doiId, doiSuffix, title);
datas.put(doiId, doiData);
}
return doiData;
}
- PipelineDoiData getDoiData(final String doiId) {
+ ProcessPipelineDoiData getDoiData(final String doiId) {
return datas.get(doiId);
}
@@ -73,18 +67,7 @@ public final class PipelineGlobalData {
}
void addNameRef(final String[] cols) {
- final String name = cols[1];
- final NameEntry entry;
- if (cols.length == 2) {
- entry = new NameEntry(name);
- } else {
- final Map<String, String> attributes = new HashMap<String, String>();
- for (int i = 2; i < cols.length; i += 2) {
- attributes.put(cols[i], cols[i + 1]);
- }
- entry = new NameEntry(name, attributes);
- }
- nameRefs.add(entry);
+ nameRefs.add(NameEntry.fromCsv(cols));
}
public TreeSet<NameEntry> getNameRefs() {
diff --git a/src/main/java/fr/osug/doi/ProcessPipelineDoiData.java b/src/main/java/fr/osug/doi/ProcessPipelineDoiData.java
index 346e794..c9ceaa5 100644
--- a/src/main/java/fr/osug/doi/ProcessPipelineDoiData.java
+++ b/src/main/java/fr/osug/doi/ProcessPipelineDoiData.java
@@ -1,5 +1,5 @@
/*******************************************************************************
- * JMMC project ( http://www.jmmc.fr ) - Copyright (C) CNRS.
+ * OSUG-DOI project ( http://doi.osug.fr ) - Copyright (C) CNRS.
******************************************************************************/
package fr.osug.doi;
@@ -12,9 +12,8 @@ import org.slf4j.LoggerFactory;
/**
*
- * @author bourgesl
*/
-public final class PipelineDoiData {
+public final class ProcessPipelineDoiData {
private final static Logger logger = LoggerFactory.getLogger(ProcessPipeline.class.getName());
@@ -31,10 +30,14 @@ public final class PipelineDoiData {
private String metadataMd5;
/** xml metadata */
private File stagingFileDOI;
+ /* data access URL */
+ private String dataAccessUrl;
/* external Landing page URL */
- private String landingPageUrl;
+ private String landingPageExtUrl;
+ /* Local Landing page URL (staging) */
+ private String landingLocUrl;
- PipelineDoiData(final String doiId, final String doiSuffix, final String title) {
+ ProcessPipelineDoiData(final String doiId, final String doiSuffix, final String title) {
this.doiId = doiId;
this.doiSuffix = doiSuffix;
this.title = title;
@@ -115,12 +118,28 @@ public final class PipelineDoiData {
this.stagingFileDOI = stagingFileDOI;
}
- public String getLandingPageUrl() {
- return landingPageUrl;
+ public String getDataAccessUrl() {
+ return dataAccessUrl;
}
- public void setLandingPageUrl(final String landingPageUrl) {
- this.landingPageUrl = landingPageUrl;
+ public void setDataAccessUrl(final String dataAccessUrl) {
+ this.dataAccessUrl = dataAccessUrl;
+ }
+
+ public String getLandingPageExtUrl() {
+ return landingPageExtUrl;
+ }
+
+ public void setLandingPageExtUrl(final String landingPageUrl) {
+ this.landingPageExtUrl = landingPageUrl;
+ }
+
+ public String getLandingLocUrl() {
+ return landingLocUrl;
+ }
+
+ public void setLandingLocUrl(final String landingLocUrl) {
+ this.landingLocUrl = landingLocUrl;
}
}
diff --git a/src/main/java/fr/osug/doi/ProjectConfig.java b/src/main/java/fr/osug/doi/ProjectConfig.java
index 3bfc08f..5fad004 100644
--- a/src/main/java/fr/osug/doi/ProjectConfig.java
+++ b/src/main/java/fr/osug/doi/ProjectConfig.java
@@ -3,16 +3,17 @@
******************************************************************************/
package fr.osug.doi;
+import fr.osug.doi.domain.NameEntry;
+import fr.osug.doi.validation.ValidationUtil;
import fr.osug.util.FileUtils;
import java.io.File;
import java.io.IOException;
import java.io.Reader;
-import java.net.MalformedURLException;
-import java.net.URL;
+import java.util.Arrays;
import java.util.Collections;
+import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
-import java.util.ListIterator;
import java.util.Map;
import java.util.Properties;
import org.slf4j.Logger;
@@ -20,57 +21,64 @@ import org.slf4j.LoggerFactory;
/**
*
- * @author bourgesl
*/
public final class ProjectConfig {
+ private final static Logger logger = LoggerFactory.getLogger(ProjectConfig.class.getName());
+
public final static String CONF_PROJECT = "project.properties";
- public final static String CONF_DOI_URL_MAPPING = "doi_url_landing_page.csv";
+ public final static String CONF_URL_MAP_DATA_ACCESS = "doi_url_data_access.csv";
+ public final static String CONF_URL_MAP_LANDING_PAGE = "doi_url_landing_page.csv";
- public final static String CONF_KEY_DATA_ACCESS_URL = "dataAccessUrl";
- public final static String CONF_KEY_GENERATE_EMBEDDED = "generate_embedded";
+ private final static String CONF_OVERRIDE_NAMES_CSV = "names_override" + Const.FILE_EXT_CSV;
- private final static File[] EMPTY = new File[0];
+ /* project.properties */
+ /** use templates for countries = "use_template_country" */
+ public final static String CONF_KEY_USE_TEMPLATE_COUNTRY = "use_template_country";
+ /** use templates for each DOI = "use_template_doi" */
+ public final static String CONF_KEY_USE_TEMPLATE_DOI = "use_template_doi";
- private final static Logger logger = LoggerFactory.getLogger(ProjectConfig.class.getName());
+ /** general data access url = "dataAccessUrl" */
+ public final static String CONF_KEY_DATA_ACCESS_URL = "dataAccessUrl";
+ /** generate embedded fragments = "generate_embedded" */
+ public final static String CONF_KEY_GENERATE_EMBEDDED = "generate_embedded";
+ /* members */
private final String projectName;
private final File projectConf;
- private final Properties properties;
-
- private final Map<String, String> landingPageUrlMapping;
-
- private final DoiTemplates templates;
- private File[] metadataCsvFiles = EMPTY;
- private File[] inputCsvFiles = EMPTY;
+ // lazy:
+ private Properties properties = null;
+ private Map<String, String> urlMapDataAccess = null;
+ private Map<String, String> urlMapLandingPages = null;
+ // Directories:
+ private File metadataDir = null;
+ private File inputDir = null;
private File tmpDir;
private File tmpDoiDir;
private File stagingDir;
- private File webStagingDir;
+ private File publicDir;
private File webStagingProjectDir;
private File webStagingProjectEmbedDir;
private File webStagingProjectXmlDir;
- private File webPublicDir;
private File webPublicProjectDir;
-
+ private File webPublicProjectEmbedDir;
+ private File webPublicProjectXmlDir;
+ // Process state:
+ private DoiTemplates templates = null;
+ private Map<String, NameEntry> overrideNameEntryMap = null;
private boolean saveCSV = false;
- public ProjectConfig(final String projectName) throws IOException {
+ public ProjectConfig(final PathConfig pathConfig, final String projectName) throws IOException {
this.projectName = projectName;
// may fail:
this.projectConf = FileUtils.getRequiredDirectory(Paths.DIR_CONFIG + projectName).getCanonicalFile();
logger.info("Project Config: {}", projectConf);
- this.properties = loadProperties(projectConf);
-
- this.landingPageUrlMapping = loadUrlMapping(new File(this.projectConf, CONF_DOI_URL_MAPPING));
-
- this.templates = new DoiTemplates(
- new File(this.projectConf, Paths.DIR_PROJECT_TEMPLATES).getAbsolutePath()
- );
+ // preload properties to ensure the folder exists:
+ getProperties();
initMetadataDir(
new File(this.projectConf, Paths.DIR_PROJECT_METADATA).getAbsolutePath()
@@ -88,11 +96,13 @@ public final class ProjectConfig {
initStagingDir(
new File(Paths.DIR_STAGING, projectName).getAbsolutePath()
);
+ initPublicDir(
+ new File(Paths.DIR_PUBLIC, projectName).getAbsolutePath()
+ );
// /www/staging directories:
- initWebStagingDir(Paths.DIR_WEB_STAGING);
initWebStagingProjectDir(
- new File(Paths.DIR_WEB_STAGING, projectName).getAbsolutePath()
+ new File(pathConfig.getWebStagingDir(), projectName).getAbsolutePath()
);
initWebStagingProjectEmbedDir(
new File(webStagingProjectDir, Paths.DIR_WEB_EMBED).getAbsolutePath()
@@ -101,19 +111,31 @@ public final class ProjectConfig {
new File(webStagingProjectDir, Paths.DIR_WEB_XML).getAbsolutePath()
);
// /www/public directories:
- initWebPublicDir(Paths.DIR_WEB_PUBLIC);
initWebPublicProjectDir(
- new File(Paths.DIR_WEB_PUBLIC, projectName).getAbsolutePath()
+ new File(pathConfig.getWebPublicDir(), projectName).getAbsolutePath()
+ );
+ initWebPublicProjectEmbedDir(
+ new File(webPublicProjectDir, Paths.DIR_WEB_EMBED).getAbsolutePath()
);
+ initWebPublicProjectXmlDir(
+ new File(webPublicProjectDir, Paths.DIR_WEB_XML).getAbsolutePath()
+ );
+ }
+
+ public void prepareProcess() throws IOException {
+ // preload needed data for the process pipeline:
+ getTemplates();
+ getUrlMapDataAccess();
+ getUrlMapLandingPage();
+ getOverrideNameMap();
}
public void initMetadataDir(final String dirPath) throws IOException {
- this.metadataCsvFiles = CsvUtil.findCSVFiles(dirPath);
+ this.metadataDir = FileUtils.getRequiredDirectory(dirPath).getCanonicalFile();
}
- // Used by junit test
public void initInputDir(final String dirPath) throws IOException {
- this.inputCsvFiles = CsvUtil.findCSVFiles(dirPath);
+ this.inputDir = FileUtils.getRequiredDirectory(dirPath).getCanonicalFile();
}
public void initTmpDir(final String path) throws IOException {
@@ -128,8 +150,8 @@ public final class ProjectConfig {
this.stagingDir = FileUtils.createDirectories(path);
}
- public void initWebStagingDir(final String path) throws IOException {
- this.webStagingDir = FileUtils.createDirectories(path);
+ public void initPublicDir(final String path) throws IOException {
+ this.publicDir = FileUtils.createDirectories(path);
}
public void initWebStagingProjectDir(final String path) throws IOException {
@@ -144,14 +166,18 @@ public final class ProjectConfig {
this.webStagingProjectXmlDir = FileUtils.createDirectories(path);
}
- public void initWebPublicDir(final String path) throws IOException {
- this.webPublicDir = FileUtils.createDirectories(path);
- }
-
public void initWebPublicProjectDir(final String path) throws IOException {
this.webPublicProjectDir = FileUtils.createDirectories(path);
}
+ public void initWebPublicProjectEmbedDir(final String path) throws IOException {
+ this.webPublicProjectEmbedDir = FileUtils.createDirectories(path);
+ }
+
+ public void initWebPublicProjectXmlDir(final String path) throws IOException {
+ this.webPublicProjectXmlDir = FileUtils.createDirectories(path);
+ }
+
public String getProjectName() {
return projectName;
}
@@ -160,32 +186,90 @@ public final class ProjectConfig {
return projectConf;
}
+ /* properties */
+ private Properties getProperties() {
+ if (properties == null) {
+ properties = loadProperties(new File(projectConf, CONF_PROJECT));
+ }
+ return properties;
+ }
+
public String getProperty(final String key) {
- final String val = properties.getProperty(key);
+ String val = getProperties().getProperty(key);
if (val != null) {
- return val.trim();
+ val = val.trim();
+ if (val.length() != 0) {
+ return val;
+ }
}
return null;
}
public boolean getPropertyBoolean(final String key) {
- return Boolean.valueOf(properties.getProperty(key, "false"));
+ return getPropertyBoolean(key, false);
}
- public String getLandingPageUrl(final String doiSuffix) {
- return landingPageUrlMapping.get(doiSuffix);
+ public boolean getPropertyBoolean(final String key, final boolean def) {
+ final String val = getProperties().getProperty(key);
+ if (val != null) {
+ return Boolean.valueOf(val);
+ }
+ return def;
+ }
+
+ /* DataAccess Url Mapping */
+ private Map<String, String> getUrlMapDataAccess() throws IOException {
+ if (urlMapDataAccess == null) {
+ urlMapDataAccess = CsvUtil.loadUrlMapping(new File(this.projectConf, CONF_URL_MAP_DATA_ACCESS));
+ }
+ return urlMapDataAccess;
+ }
+
+ public String getUrlDataAccess(final String doiSuffix) throws IOException {
+ return getUrlMapDataAccess().get(doiSuffix);
+ }
+
+ /* LandingPage Url Mapping */
+ private Map<String, String> getUrlMapLandingPage() throws IOException {
+ if (urlMapLandingPages == null) {
+ urlMapLandingPages = CsvUtil.loadUrlMapping(new File(this.projectConf, CONF_URL_MAP_LANDING_PAGE));
+ }
+ return urlMapLandingPages;
+ }
+
+ public String getUrlLandingPage(final String doiSuffix) throws IOException {
+ return getUrlMapLandingPage().get(doiSuffix);
}
- public DoiTemplates getTemplates() {
+ /* templates */
+ public DoiTemplates getTemplates() throws IOException {
+ if (templates == null) {
+ templates = new DoiTemplates(
+ new File(this.projectConf, Paths.DIR_PROJECT_TEMPLATES).getAbsolutePath()
+ );
+ }
return templates;
}
- public File[] getMetadataCsvFiles() {
- return metadataCsvFiles;
+ /* names override */
+ private Map<String, NameEntry> getOverrideNameMap() throws IOException {
+ if (overrideNameEntryMap == null) {
+ overrideNameEntryMap = loadOverrideNames(new File(this.projectConf, CONF_OVERRIDE_NAMES_CSV));
+ }
+ return overrideNameEntryMap;
+ }
+
+ public NameEntry getOverrideNameEntry(final String simpleName) throws IOException {
+ return getOverrideNameMap().get(simpleName);
+ }
+
+ /* Directories */
+ public File getMetadataDir() {
+ return metadataDir;
}
- public File[] getInputCsvFiles() {
- return inputCsvFiles;
+ public File getInputDir() {
+ return inputDir;
}
public File getTmpDir() {
@@ -200,8 +284,8 @@ public final class ProjectConfig {
return stagingDir;
}
- public File getWebStagingDir() {
- return webStagingDir;
+ public File getPublicDir() {
+ return publicDir;
}
public File getWebStagingProjectDir() {
@@ -216,14 +300,18 @@ public final class ProjectConfig {
return webStagingProjectXmlDir;
}
- public File getWebPublicDir() {
- return webPublicDir;
- }
-
public File getWebPublicProjectDir() {
return webPublicProjectDir;
}
+ public File getWebPublicProjectEmbedDir() {
+ return webPublicProjectEmbedDir;
+ }
+
+ public File getWebPublicProjectXmlDir() {
+ return webPublicProjectXmlDir;
+ }
+
public boolean isSaveCSV() {
return saveCSV;
}
@@ -232,8 +320,7 @@ public final class ProjectConfig {
this.saveCSV = saveCSV;
}
- private static Properties loadProperties(final File projectConf) {
- final File propFile = new File(projectConf, CONF_PROJECT);
+ private static Properties loadProperties(final File propFile) {
logger.info("Loading {}", propFile);
final Properties props = new Properties();
@@ -250,44 +337,82 @@ public final class ProjectConfig {
return props;
}
- private static Map<String, String> loadUrlMapping(final File csvFile) throws IOException {
- final Map<String, String> urlMapping;
+ private static Map<String, NameEntry> loadOverrideNames(final File csvFile) throws IOException {
+ Map<String, NameEntry> nameEntryMap = Collections.emptyMap();
if (csvFile.exists()) {
final CsvData data = CsvUtil.read(csvFile);
logger.debug("CSV data: {}", data);
final List<String[]> rows = data.getRows();
-
- urlMapping = new LinkedHashMap<String, String>(rows.size());
-
- // Get mappings [DOI|URL]
- for (ListIterator<String[]> it = rows.listIterator(); it.hasNext();) {
- final String[] cols = it.next();
- if (cols == null || cols.length != 2) {
- // remove line:
- it.remove();
- continue;
- }
- String key = cols[0].trim();
- if (key.isEmpty() || key.charAt(0) == Const.COMMENT) {
- continue;
- }
- String url = cols[1].trim();
- if (url.isEmpty() || !url.startsWith("http")) {
- continue;
- }
- try {
- // should detect repeated keys or values ?
- urlMapping.put(key, new URL(url).toString());
- } catch (MalformedURLException mue) {
- logger.info("invalid URL: {}", url);
+ final int nbRows = rows.size();
+
+ if (nbRows > 1) {
+ // Parse header:
+ String[] keys = null;
+ String[] cols = rows.get(0);
+
+ if (cols == null || cols.length < 1) {
+ logger.error("Missing header in {}", csvFile);
+ } else {
+ keys = cols;
+ // Check keys:
+ for (int i = 0; i < keys.length; i++) {
+ String key = keys[i];
+
+ if (key.length() > 1) {
+ if (key.charAt(0) == Const.COMMENT) {
+ key = key.substring(1);
+ }
+ key = key.trim();
+ }
+ if (key.isEmpty()) {
+ logger.error("Missing column name at {}", i);
+ key = null; // ignore
+ }
+ keys[i] = key;
+ }
+
+ if (logger.isDebugEnabled()) {
+ logger.debug("keys: {}", Arrays.toString(keys));
+ }
+
+ nameEntryMap = new LinkedHashMap<String, NameEntry>(nbRows - 1);
+
+ // Get mappings [Name|...]
+ for (int i = 1; i < nbRows; i++) {
+ cols = rows.get(i);
+
+ if (cols == null || cols.length < 1) {
+ continue;
+ }
+ String name = cols[0].trim();
+ if (name.isEmpty() || name.charAt(0) == Const.COMMENT) {
+ continue;
+ }
+
+ Map<String, String> attributes = null;
+
+ if (cols.length > 1) {
+ attributes = new HashMap<String, String>(4);
+
+ for (int j = 1; j < cols.length; j++) {
+ String key = keys[j];
+ String val = cols[j];
+ if (key != null && !val.trim().isEmpty()) {
+ attributes.put(key, val);
+ }
+ }
+ }
+
+ final String simpleName = ValidationUtil.simplifyName(name);
+
+ nameEntryMap.put(simpleName, new NameEntry(name, attributes));
+ }
}
}
- } else {
- urlMapping = Collections.emptyMap();
}
- logger.info("urlMapping: {}", urlMapping);
- return urlMapping;
+ logger.debug("nameEntryMap: {}", nameEntryMap);
+ return nameEntryMap;
}
}
diff --git a/src/main/java/fr/osug/doi/domain/Doi.java b/src/main/java/fr/osug/doi/domain/Doi.java
index 2cfd58c..2d509b0 100644
--- a/src/main/java/fr/osug/doi/domain/Doi.java
+++ b/src/main/java/fr/osug/doi/domain/Doi.java
@@ -26,7 +26,7 @@ public class Doi implements Persistable<Long> {
private static final long serialVersionUID = 1L;
@Id
- @SequenceGenerator(name = "doi_id_seq", sequenceName = "doi_id_seq", allocationSize = 10)
+ @SequenceGenerator(name = "doi_id_seq", sequenceName = "doi_id_seq", initialValue = 1, allocationSize = 10)
@GeneratedValue(strategy = GenerationType.SEQUENCE, generator = "doi_id_seq")
private Long id;
@@ -36,12 +36,15 @@ public class Doi implements Persistable<Long> {
@Column(name = "identifier", unique = true, nullable = false)
private String identifier;
+ @Column(name = "description")
+ private String description;
+
@Enumerated(EnumType.STRING)
@Column(name = "status")
private Status status = Status.STAGING;
- @Column(name = "description")
- private String description;
+ @Column(name = "data_access_url")
+ private String dataAccessUrl;
@Column(name = "landing_ext_url")
private String landingExtUrl;
@@ -92,6 +95,14 @@ public class Doi implements Persistable<Long> {
this.status = status;
}
+ public String getDataAccessUrl() {
+ return dataAccessUrl;
+ }
+
+ public void setDataAccessUrl(String dataAccessUrl) {
+ this.dataAccessUrl = dataAccessUrl;
+ }
+
public String getLandingExtUrl() {
return landingExtUrl;
}
@@ -101,23 +112,28 @@ public class Doi implements Persistable<Long> {
}
@Override
- public boolean equals(Object o) {
- if (this == o) {
+ public int hashCode() {
+ int hash = 3;
+ hash = 17 * hash + Objects.hashCode(this.identifier);
+ return hash;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
return true;
}
- if (o == null || getClass() != o.getClass()) {
+ if (obj == null) {
return false;
}
- Doi doi = (Doi) o;
- if (doi.getId() == null || id == null) {
+ if (getClass() != obj.getClass()) {
return false;
}
- return Objects.equals(id, doi.getId());
- }
-
- @Override
- public int hashCode() {
- return Objects.hashCode(id);
+ final Doi other = (Doi) obj;
+ if (!Objects.equals(this.identifier, other.identifier)) {
+ return false;
+ }
+ return true;
}
@Override
@@ -128,6 +144,7 @@ public class Doi implements Persistable<Long> {
+ ", identifier='" + identifier + "'"
+ ", description='" + description + "'"
+ ", status='" + status + "'"
+ + ", dataAccessUrl='" + dataAccessUrl + "'"
+ ", landingExtUrl='" + landingExtUrl + "'"
+ '}';
}
diff --git a/src/main/java/fr/osug/doi/domain/DoiCommon.java b/src/main/java/fr/osug/doi/domain/DoiCommon.java
new file mode 100644
index 0000000..aba2e3c
--- /dev/null
+++ b/src/main/java/fr/osug/doi/domain/DoiCommon.java
@@ -0,0 +1,139 @@
+/*******************************************************************************
+ * OSUG-DOI project ( http://doi.osug.fr ) - Copyright (C) CNRS.
+ ******************************************************************************/
+package fr.osug.doi.domain;
+
+import java.util.Date;
+import java.util.Objects;
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.Id;
+import javax.persistence.Inheritance;
+import javax.persistence.InheritanceType;
+import javax.persistence.JoinColumn;
+import javax.persistence.OneToOne;
+import javax.persistence.SequenceGenerator;
+import javax.persistence.Temporal;
+import javax.persistence.TemporalType;
+import org.springframework.data.domain.Persistable;
+
+/**
+ *
+ */
+@Entity
+@Inheritance(strategy = InheritanceType.TABLE_PER_CLASS)
+public abstract class DoiCommon implements Persistable<Long> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Id
+ @Column(name = "id")
+ @SequenceGenerator(name = "doi_common_id_seq", sequenceName = "doi_common_id_seq", initialValue = 1, allocationSize = 10)
+ @GeneratedValue(strategy = GenerationType.SEQUENCE, generator = "doi_common_id_seq")
+ private Long id;
+
+ @OneToOne
+ @JoinColumn(name = "doi_id")
+ private Doi doi;
+
+ @Column(name = "metadata_md5")
+ private String metadataMd5;
+
+ @Column(name = "landing_loc_url")
+ private String landingLocUrl;
+
+ @Column(name = "create_date")
+ @Temporal(TemporalType.TIMESTAMP)
+ private Date createDate;
+
+ @Column(name = "update_date")
+ @Temporal(TemporalType.TIMESTAMP)
+ private Date updateDate;
+
+ @Override
+ public final Long getId() {
+ return id;
+ }
+
+ @Override
+ public final boolean isNew() {
+ return id == null;
+ }
+
+ public final Doi getDoi() {
+ return doi;
+ }
+
+ public final void setDoi(Doi doi) {
+ this.doi = doi;
+ }
+
+ public final String getMetadataMd5() {
+ return metadataMd5;
+ }
+
+ public final void setMetadataMd5(final String metadataMd5) {
+ this.metadataMd5 = metadataMd5;
+ }
+
+ public final String getLandingLocUrl() {
+ return landingLocUrl;
+ }
+
+ public final void setLandingLocUrl(final String landingLocUrl) {
+ this.landingLocUrl = landingLocUrl;
+ }
+
+ public final Date getCreateDate() {
+ return createDate;
+ }
+
+ public final void setCreateDate(Date createDate) {
+ this.createDate = createDate;
+ }
+
+ public final Date getUpdateDate() {
+ return updateDate;
+ }
+
+ public final void setUpdateDate(final Date updateDate) {
+ this.updateDate = updateDate;
+ }
+
+ @Override
+ public int hashCode() {
+ int hash = 7;
+ hash = 47 * hash + Objects.hashCode(this.doi);
+ return hash;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ final DoiCommon other = (DoiCommon) obj;
+ if (!Objects.equals(this.doi, other.doi)) {
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public String toString() {
+ return "id=" + id
+ + ", doi=[" + doi + "]"
+ + ", metadataMd5='" + metadataMd5 + "'"
+ + ", landingLocUrl='" + landingLocUrl + "'"
+ + ", createDate='" + createDate + "'"
+ + ", updateDate='" + updateDate + "'";
+ }
+}
diff --git a/src/main/java/fr/osug/doi/domain/DoiPublic.java b/src/main/java/fr/osug/doi/domain/DoiPublic.java
index be8d3a4..1e1c59a 100644
--- a/src/main/java/fr/osug/doi/domain/DoiPublic.java
+++ b/src/main/java/fr/osug/doi/domain/DoiPublic.java
@@ -3,91 +3,22 @@
******************************************************************************/
package fr.osug.doi.domain;
-import java.util.Date;
-import java.util.Objects;
import javax.persistence.Column;
import javax.persistence.Entity;
-import javax.persistence.GeneratedValue;
-import javax.persistence.GenerationType;
-import javax.persistence.Id;
-import javax.persistence.JoinColumn;
-import javax.persistence.OneToOne;
-import javax.persistence.SequenceGenerator;
import javax.persistence.Table;
-import javax.persistence.Temporal;
-import javax.persistence.TemporalType;
-import org.springframework.data.domain.Persistable;
/**
* A Doi in the public phase
*/
@Entity
@Table(name = "doi_public")
-public class DoiPublic implements Persistable<Long> {
+public class DoiPublic extends DoiCommon {
private static final long serialVersionUID = 1L;
- @Id
- @Column(name = "doi_id")
- @SequenceGenerator(name = "doi_public_id_seq", sequenceName = "doi_public_id_seq", allocationSize = 10)
- @GeneratedValue(strategy = GenerationType.SEQUENCE, generator = "doi_public_id_seq")
- private Long id;
-
- @OneToOne
- @JoinColumn(name = "doi_id")
- private Doi doi;
-
- @Column(name = "metadata_md5")
- private String metadataMd5;
-
- @Column(name = "landing_loc_url")
- private String landingLocUrl;
-
@Column(name = "active_ext_url")
private boolean activeExtUrl;
- @Column(name = "create_date")
- @Temporal(TemporalType.TIMESTAMP)
- private Date createDate;
-
- @Column(name = "update_date")
- @Temporal(TemporalType.TIMESTAMP)
- private Date updateDate;
-
- @Override
- public Long getId() {
- return id;
- }
-
- @Override
- public boolean isNew() {
- return id == null;
- }
-
- public Doi getDoi() {
- return doi;
- }
-
- public void setDoi(Doi doi) {
- this.doi = doi;
- }
-
- public String getMetadataMd5() {
- return metadataMd5;
- }
-
- public void setMetadataMd5(String metadataMd5) {
- this.metadataMd5 = metadataMd5;
- }
-
- public String getLandingLocUrl() {
- return landingLocUrl;
- }
-
- public void setLandingLocUrl(String landingLocUrl) {
- this.landingLocUrl = landingLocUrl;
- }
-
public boolean isActiveExtUrl() {
return activeExtUrl;
}
@@ -96,50 +27,11 @@ public class DoiPublic implements Persistable<Long> {
this.activeExtUrl = activeExtUrl;
}
- public Date getCreateDate() {
- return createDate;
- }
-
- public void setCreateDate(Date createDate) {
- this.createDate = createDate;
- }
-
- public Date getUpdateDate() {
- return updateDate;
- }
-
- public void setUpdateDate(Date updateDate) {
- this.updateDate = updateDate;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- DoiPublic doi = (DoiPublic) o;
- if (doi.getId() == null || getId() == null) {
- return false;
- }
- return Objects.equals(getId(), doi.getId());
- }
-
- @Override
- public int hashCode() {
- return Objects.hashCode(getId());
- }
-
@Override
public String toString() {
return "DoiPublic{"
- + "doi=" + doi
+ + super.toString()
+ ", activeExtUrl='" + activeExtUrl + "'"
- + ", landingLocUrl='" + landingLocUrl + "'"
- + ", createDate='" + createDate + "'"
- + ", updateDate='" + updateDate + "'"
+ '}';
}
}
diff --git a/src/main/java/fr/osug/doi/domain/DoiStaging.java b/src/main/java/fr/osug/doi/domain/DoiStaging.java
index a06d8f1..b348a19 100644
--- a/src/main/java/fr/osug/doi/domain/DoiStaging.java
+++ b/src/main/java/fr/osug/doi/domain/DoiStaging.java
@@ -3,77 +3,25 @@
******************************************************************************/
package fr.osug.doi.domain;
-import java.util.Date;
-import java.util.Objects;
import javax.persistence.Column;
import javax.persistence.Entity;
-import javax.persistence.GeneratedValue;
-import javax.persistence.GenerationType;
-import javax.persistence.Id;
-import javax.persistence.JoinColumn;
-import javax.persistence.OneToOne;
-import javax.persistence.SequenceGenerator;
import javax.persistence.Table;
-import javax.persistence.Temporal;
-import javax.persistence.TemporalType;
-import org.springframework.data.domain.Persistable;
/**
* A Doi in the staging phase
*/
@Entity
@Table(name = "doi_staging")
-public class DoiStaging implements Persistable<Long> {
+public class DoiStaging extends DoiCommon {
private static final long serialVersionUID = 1L;
- @Id
- @SequenceGenerator(name = "doi_staging_id_seq", sequenceName = "doi_staging_id_seq", allocationSize = 10)
- @GeneratedValue(strategy = GenerationType.SEQUENCE, generator = "doi_staging_id_seq")
- private Long id;
-
- @OneToOne
- @JoinColumn(name = "doi_id")
- private Doi doi;
-
@Column(name = "valid")
private boolean valid;
- @Column(name = "metadata_md5")
- private String metadataMd5;
-
@Column(name = "log")
private String log;
- @Column(name = "landing_loc_url")
- private String landingLocUrl;
-
- @Column(name = "create_date")
- @Temporal(TemporalType.TIMESTAMP)
- private Date createDate;
-
- @Column(name = "update_date")
- @Temporal(TemporalType.TIMESTAMP)
- private Date updateDate;
-
- @Override
- public Long getId() {
- return id;
- }
-
- @Override
- public boolean isNew() {
- return id == null;
- }
-
- public Doi getDoi() {
- return doi;
- }
-
- public void setDoi(Doi doi) {
- this.doi = doi;
- }
-
public boolean isValid() {
return valid;
}
@@ -82,14 +30,6 @@ public class DoiStaging implements Persistable<Long> {
this.valid = valid;
}
- public String getMetadataMd5() {
- return metadataMd5;
- }
-
- public void setMetadataMd5(String metadataMd5) {
- this.metadataMd5 = metadataMd5;
- }
-
public String getLog() {
return log;
}
@@ -98,60 +38,11 @@ public class DoiStaging implements Persistable<Long> {
this.log = log;
}
- public String getLandingLocUrl() {
- return landingLocUrl;
- }
-
- public void setLandingLocUrl(String landingLocUrl) {
- this.landingLocUrl = landingLocUrl;
- }
-
- public Date getCreateDate() {
- return createDate;
- }
-
- public void setCreateDate(Date createDate) {
- this.createDate = createDate;
- }
-
- public Date getUpdateDate() {
- return updateDate;
- }
-
- public void setUpdateDate(Date updateDate) {
- this.updateDate = updateDate;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- DoiStaging doi = (DoiStaging) o;
- if (doi.getId() == null || getId() == null) {
- return false;
- }
- return Objects.equals(getId(), doi.getId());
- }
-
- @Override
- public int hashCode() {
- return Objects.hashCode(getId());
- }
-
@Override
public String toString() {
return "DoiStaging{"
- + "id=" + id
- + ", doi=" + doi
+ + super.toString()
+ ", valid='" + valid + "'"
- + ", metadataMd5='" + metadataMd5 + "'"
- + ", landingLocUrl='" + landingLocUrl + "'"
- + ", createDate='" + createDate + "'"
- + ", updateDate='" + updateDate + "'"
+ ", log=[\n" + log + "\n]"
+ '}';
}
diff --git a/src/main/java/fr/osug/doi/domain/IndexEntry.java b/src/main/java/fr/osug/doi/domain/IndexEntry.java
index c7f5db5..5867ffb 100644
--- a/src/main/java/fr/osug/doi/domain/IndexEntry.java
+++ b/src/main/java/fr/osug/doi/domain/IndexEntry.java
@@ -1,5 +1,5 @@
/*******************************************************************************
- * JMMC project ( http://www.jmmc.fr ) - Copyright (C) CNRS.
+ * OSUG-DOI project ( http://doi.osug.fr ) - Copyright (C) CNRS.
******************************************************************************/
package fr.osug.doi.domain;
@@ -7,7 +7,6 @@ import java.util.Date;
/**
*
- * @author bourgesl
*/
public final class IndexEntry {
diff --git a/src/main/java/fr/osug/doi/domain/NameEntry.java b/src/main/java/fr/osug/doi/domain/NameEntry.java
index 478b863..92d33f1 100644
--- a/src/main/java/fr/osug/doi/domain/NameEntry.java
+++ b/src/main/java/fr/osug/doi/domain/NameEntry.java
@@ -1,15 +1,16 @@
/*******************************************************************************
- * JMMC project ( http://www.jmmc.fr ) - Copyright (C) CNRS.
+ * OSUG-DOI project ( http://doi.osug.fr ) - Copyright (C) CNRS.
******************************************************************************/
package fr.osug.doi.domain;
import java.util.Comparator;
+import java.util.HashMap;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Objects;
/**
*
- * @author bourgesl
*/
public final class NameEntry {
@@ -26,15 +27,42 @@ public final class NameEntry {
}
};
- private final String name;
- private final Map<String, String> attributes;
+ public static NameEntry fromCsv(final String[] cols) {
+ final String name = cols[1];
+ final NameEntry entry;
+ if (cols.length == 2) {
+ entry = new NameEntry(name, null);
+ } else {
+ final Map<String, String> attributes = new HashMap<String, String>(4);
+ for (int i = 2; i < cols.length; i += 2) {
+ attributes.put(cols[i], cols[i + 1]);
+ }
+ entry = new NameEntry(name, attributes);
+ }
+ return entry;
+ }
- public NameEntry(String name) {
- this.name = name;
- this.attributes = null;
+ public static String[] toCsv(final NameEntry entry) {
+ if (!entry.hasAttributes()) {
+ return new String[]{null, entry.getName()};
+ }
+ final Map<String, String> attributes = entry.getAttributes();
+ final String[] cols = new String[(1 + attributes.size()) * 2];
+ cols[1] = entry.getName();
+ int i = 2;
+ for (Entry<String, String> e : attributes.entrySet()) {
+ cols[i] = e.getKey();
+ cols[i + 1] = e.getValue();
+ i += 2;
+ }
+ return cols;
}
- public NameEntry(String name, Map<String, String> attributes) {
+ // members:
+ private final String name;
+ private final Map<String, String> attributes;
+
+ public NameEntry(final String name, final Map<String, String> attributes) {
this.name = name;
this.attributes = attributes;
}
diff --git a/src/main/java/fr/osug/doi/domain/Project.java b/src/main/java/fr/osug/doi/domain/Project.java
index c43210b..e202af3 100644
--- a/src/main/java/fr/osug/doi/domain/Project.java
+++ b/src/main/java/fr/osug/doi/domain/Project.java
@@ -26,7 +26,7 @@ public class Project implements Persistable<Long> {
private static final long serialVersionUID = 1L;
@Id
- @SequenceGenerator(name = "project_id_seq", sequenceName = "project_id_seq", allocationSize = 1)
+ @SequenceGenerator(name = "project_id_seq", sequenceName = "project_id_seq", initialValue = 1, allocationSize = 10)
@GeneratedValue(strategy = GenerationType.SEQUENCE, generator = "project_id_seq")
private Long id;
@@ -102,23 +102,28 @@ public class Project implements Persistable<Long> {
}
@Override
- public boolean equals(Object o) {
- if (this == o) {
+ public int hashCode() {
+ int hash = 7;
+ hash = 97 * hash + Objects.hashCode(this.name);
+ return hash;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
return true;
}
- if (o == null || getClass() != o.getClass()) {
+ if (obj == null) {
return false;
}
- Project project = (Project) o;
- if (project.getId() == null || id == null) {
+ if (getClass() != obj.getClass()) {
return false;
}
- return Objects.equals(id, project.getId());
- }
-
- @Override
- public int hashCode() {
- return Objects.hashCode(id);
+ final Project other = (Project) obj;
+ if (!Objects.equals(this.name, other.name)) {
+ return false;
+ }
+ return true;
}
@Override
diff --git a/src/main/java/fr/osug/doi/domain/model/dbmodel.jpa b/src/main/java/fr/osug/doi/domain/model/dbmodel.jpa
index 2feb2c4..39d6ec2 100644
--- a/src/main/java/fr/osug/doi/domain/model/dbmodel.jpa
+++ b/src/main/java/fr/osug/doi/domain/model/dbmodel.jpa
@@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<jpa:entity-mappings xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:java="http://jcp.org/en/jsr/detail?id=270" xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:jpa="http://java.sun.com/xml/ns/persistence/orm" v="2.6.5" status="GENERATED" sm="false" xs="false" id="_1481192659941462" xsi:schemaLocation="http://java.sun.com/xml/ns/persistence/orm orm_2_1.xsd">
- <jpa:entity xre="false" abs="false" class="Project" visibile="true" minimized="false" rootElement="_1481192659941462" id="_1481192659941463">
+<jpa:entity-mappings xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:java="http://jcp.org/en/jsr/detail?id=270" xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:jpa="http://java.sun.com/xml/ns/persistence/orm" v="2.6.5" sm="false" xs="false" id="_1481192659941462" xsi:schemaLocation="http://java.sun.com/xml/ns/persistence/orm orm_2_1.xsd">
+ <jpa:entity xre="false" compositePrimaryKeyClass="ProjectPK" abs="false" class="Project" visibile="true" minimized="false" rootElement="_1481192659941462" id="_1481192659941463">
<jpa:attributes>
<jpa:basic optional="false" attribute-type="String" visibile="true" name="name" id="_1481192659941468">
<jpa:column name="name" unique="false" nullable="false" insertable="true" updatable="true" table="project" length="32"/>
@@ -19,7 +19,7 @@
<jpa:temporal>TIMESTAMP</jpa:temporal>
<jpa:column name="update_date" unique="false" nullable="true" insertable="true" updatable="true" table="project"/>
</jpa:basic>
- <jpa:one-to-many own="false" connected-entity-id="_1481192659941464" connected-attribute-id="_1481192659941479" visibile="true" name="doiCollection" id="_1481192659941473"/>
+ <jpa:one-to-many own="false" collection-type="java.util.List" connected-entity-id="_1481192659941464" connected-attribute-id="_1481192659941479" visibile="true" name="doiCollection" id="_1481192659941473"/>
<jpa:id attribute-type="Long" visibile="true" name="id" id="_1481192659941467">
<jpa:column name="id" unique="false" nullable="false" insertable="true" updatable="true" table="project"/>
</jpa:id>
@@ -30,7 +30,7 @@
</jpa:unique-constraint>
</jpa:table>
</jpa:entity>
- <jpa:entity xre="false" abs="false" class="Doi" visibile="true" minimized="false" rootElement="_1481192659941462" id="_1481192659941464">
+ <jpa:entity xre="false" compositePrimaryKeyClass="DoiPK" abs="false" class="Doi" visibile="true" minimized="false" rootElement="_1481192659941462" id="_1481192659941464">
<jpa:attributes>
<jpa:basic optional="false" attribute-type="String" visibile="true" name="identifier" id="_1481192659941475">
<jpa:column name="identifier" unique="false" nullable="false" insertable="true" updatable="true" table="doi" length="255"/>
@@ -44,11 +44,11 @@
<jpa:basic optional="true" attribute-type="String" visibile="true" name="landingExtUrl" id="_1481192659941478">
<jpa:column name="landing_ext_url" unique="false" nullable="true" insertable="true" updatable="true" table="doi" length="1024"/>
</jpa:basic>
- <jpa:many-to-one optional="false" connected-entity-id="_1481192659941463" connected-attribute-id="_1481192659941473" visibile="true" name="projectId" id="_1481192659941479">
+ <jpa:many-to-one optional="false" primaryKey="false" connected-entity-id="_1481192659941463" connected-attribute-id="_1481192659941473" visibile="true" name="projectId" id="_1481192659941479">
<jpa:join-column name="PROJECT_ID" rc="ID" unique="false" nullable="true" insertable="true" updatable="true"/>
</jpa:many-to-one>
- <jpa:one-to-many own="false" connected-entity-id="_1481192659941465" connected-attribute-id="_1481192659941488" visibile="true" name="doiPublicCollection" id="_1481192659941480"/>
- <jpa:one-to-many own="false" connected-entity-id="_1481192659941466" connected-attribute-id="_1481192659941496" visibile="true" name="doiStagingCollection" id="_1481192659941481"/>
+ <jpa:one-to-many own="false" collection-type="java.util.List" connected-entity-id="_1481192659941465" connected-attribute-id="_1481192659941488" visibile="true" name="doiPublicCollection" id="_1481192659941480"/>
+ <jpa:one-to-many own="false" collection-type="java.util.List" connected-entity-id="_1481192659941466" connected-attribute-id="_1481192659941496" visibile="true" name="doiStagingCollection" id="_1481192659941481"/>
<jpa:id attribute-type="Long" visibile="true" name="id" id="_1481192659941474">
<jpa:column name="id" unique="false" nullable="false" insertable="true" updatable="true" table="doi"/>
</jpa:id>
@@ -59,7 +59,7 @@
</jpa:unique-constraint>
</jpa:table>
</jpa:entity>
- <jpa:entity xre="false" abs="false" class="DoiPublic" visibile="true" minimized="false" rootElement="_1481192659941462" id="_1481192659941465">
+ <jpa:entity xre="false" compositePrimaryKeyClass="DoiPublicPK" abs="false" class="DoiPublic" visibile="true" minimized="false" rootElement="_1481192659941462" id="_1481192659941465">
<jpa:attributes>
<jpa:basic optional="false" attribute-type="String" visibile="true" name="metadataMd5" id="_1481192659941483">
<jpa:column name="metadata_md5" unique="false" nullable="false" insertable="true" updatable="true" table="doi_public" length="32"/>
@@ -78,7 +78,7 @@
<jpa:temporal>TIMESTAMP</jpa:temporal>
<jpa:column name="update_date" unique="false" nullable="true" insertable="true" updatable="true" table="doi_public"/>
</jpa:basic>
- <jpa:many-to-one optional="false" connected-entity-id="_1481192659941464" connected-attribute-id="_1481192659941480" visibile="true" name="doiId" id="_1481192659941488">
+ <jpa:many-to-one optional="false" primaryKey="false" connected-entity-id="_1481192659941464" connected-attribute-id="_1481192659941480" visibile="true" name="doiId" id="_1481192659941488">
<jpa:join-column name="DOI_ID" rc="ID" unique="false" nullable="true" insertable="true" updatable="true"/>
</jpa:many-to-one>
<jpa:id attribute-type="Long" visibile="true" name="id" id="_1481192659941482">
@@ -87,7 +87,7 @@
</jpa:attributes>
<jpa:table name="doi_public"/>
</jpa:entity>
- <jpa:entity xre="false" abs="false" class="DoiStaging" visibile="true" minimized="false" rootElement="_1481192659941462" id="_1481192659941466">
+ <jpa:entity xre="false" compositePrimaryKeyClass="DoiStagingPK" abs="false" class="DoiStaging" visibile="true" minimized="false" rootElement="_1481192659941462" id="_1481192659941466">
<jpa:attributes>
<jpa:basic optional="true" attribute-type="Boolean" visibile="true" name="valid" id="_1481192659941490">
<jpa:column name="valid" unique="false" nullable="true" insertable="true" updatable="true" table="doi_staging"/>
@@ -109,7 +109,7 @@
<jpa:temporal>TIMESTAMP</jpa:temporal>
<jpa:column name="update_date" unique="false" nullable="true" insertable="true" updatable="true" table="doi_staging"/>
</jpa:basic>
- <jpa:many-to-one optional="false" connected-entity-id="_1481192659941464" connected-attribute-id="_1481192659941481" visibile="true" name="doiId" id="_1481192659941496">
+ <jpa:many-to-one optional="false" primaryKey="false" connected-entity-id="_1481192659941464" connected-attribute-id="_1481192659941481" visibile="true" name="doiId" id="_1481192659941496">
<jpa:join-column name="DOI_ID" rc="ID" unique="false" nullable="true" insertable="true" updatable="true"/>
</jpa:many-to-one>
<jpa:id attribute-type="Long" visibile="true" name="id" id="_1481192659941489">
@@ -122,7 +122,21 @@
* This file was generated by the JPA Modeler
*/</jpa:snp>
<jpa:inf e="true" n="java.io.Serializable"/>
+ <jpa:c/>
<jpa:diagram>
- <plane elementRef="_1481192659941462"/>
+ <plane elementRef="_1481192659941462">
+ <shape elementRef="_1481192659941463">
+ <Bounds x="32.0" y="576.0" width="146.0" height="198.0"/>
+ </shape>
+ <shape elementRef="_1481192659941464">
+ <Bounds x="257.0" y="294.0" width="185.0" height="218.0"/>
+ </shape>
+ <shape elementRef="_1481192659941465">
+ <Bounds x="32.0" y="32.0" width="161.0" height="198.0"/>
+ </shape>
+ <shape elementRef="_1481192659941466">
+ <Bounds x="506.0" y="576.0" width="174.0" height="218.0"/>
+ </shape>
+ </plane>
</jpa:diagram>
</jpa:entity-mappings>
diff --git a/src/main/java/fr/osug/doi/repository/DoiBaseRepository.java b/src/main/java/fr/osug/doi/repository/DoiBaseRepository.java
index f4a91f1..fd73914 100644
--- a/src/main/java/fr/osug/doi/repository/DoiBaseRepository.java
+++ b/src/main/java/fr/osug/doi/repository/DoiBaseRepository.java
@@ -4,34 +4,29 @@
package fr.osug.doi.repository;
import fr.osug.doi.domain.Doi;
-import fr.osug.doi.domain.DoiStaging;
+import fr.osug.doi.domain.DoiCommon;
import java.util.List;
-import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Query;
+import org.springframework.data.repository.NoRepositoryBean;
+import org.springframework.data.repository.Repository;
import org.springframework.data.repository.query.Param;
-import org.springframework.stereotype.Repository;
import org.springframework.transaction.annotation.Transactional;
/**
- * Spring Data JPA repository for the DoiStaging entity.
+ * Spring Data JPA repository for the DoiCommon classes.
*/
-@Repository
-@Transactional
-public interface DoiStagingRepository extends JpaRepository<DoiStaging, Long> {
+@NoRepositoryBean
+public interface DoiBaseRepository<T extends DoiCommon> extends Repository<T, Long> {
@Transactional(readOnly = true)
- DoiStaging findOneByDoi(Doi doi);
+ T findOneByDoi(Doi doi);
@Transactional(readOnly = true)
- @Query("select ds from DoiStaging ds join ds.doi d join d.project p where (p.name = :projectName) order by d.identifier asc")
- List<DoiStaging> findByProject(@Param("projectName") String projectName);
-
- @Transactional(readOnly = true)
- @Query("select ds from DoiStaging ds join ds.doi d join d.project p where (p.name = :projectName) and ds.valid = false order by d.identifier asc")
- List<DoiStaging> findErrorByProject(@Param("projectName") String projectName);
+ @Query("select ds from #{#entityName} ds join ds.doi d join d.project p where (p.name = :projectName) order by d.identifier asc")
+ List<T> findByProject(@Param("projectName") String projectName);
@Transactional(readOnly = true)
- @Query("select count(ds) from DoiStaging ds join ds.doi d join d.project p where (p.name = :projectName)")
+ @Query("select count(ds) from #{#entityName} ds join ds.doi d join d.project p where (p.name = :projectName)")
Long countByProject(@Param("projectName") String projectName);
}
diff --git a/src/main/java/fr/osug/doi/repository/DoiPublicRepository.java b/src/main/java/fr/osug/doi/repository/DoiPublicRepository.java
index 6cc0897..0d73034 100644
--- a/src/main/java/fr/osug/doi/repository/DoiPublicRepository.java
+++ b/src/main/java/fr/osug/doi/repository/DoiPublicRepository.java
@@ -3,28 +3,16 @@
******************************************************************************/
package fr.osug.doi.repository;
-import fr.osug.doi.domain.Doi;
import fr.osug.doi.domain.DoiPublic;
-import fr.osug.doi.domain.DoiStaging;
-import java.util.List;
import org.springframework.data.jpa.repository.JpaRepository;
-import org.springframework.data.jpa.repository.Query;
-import org.springframework.data.repository.query.Param;
import org.springframework.stereotype.Repository;
import org.springframework.transaction.annotation.Transactional;
/**
- * Spring Data JPA repository for the DoiStaging entity.
+ * Spring Data JPA repository for the DoiPublic entity.
*/
@Repository
@Transactional
-public interface DoiPublicRepository extends JpaRepository<DoiPublic, Long> {
-
- @Transactional(readOnly = true)
- DoiPublic findOneByDoi(Doi doi);
-
- @Transactional(readOnly = true)
- @Query("select dp from DoiPublic dp join dp.doi d join d.project p where (p.name = :projectName) order by d.identifier asc")
- List<DoiPublic> findByProject(@Param("projectName") String projectName);
+public interface DoiPublicRepository extends DoiBaseRepository<DoiPublic>, JpaRepository<DoiPublic, Long> {
}
diff --git a/src/main/java/fr/osug/doi/repository/DoiStagingRepository.java b/src/main/java/fr/osug/doi/repository/DoiStagingRepository.java
index f4a91f1..6c4f3ee 100644
--- a/src/main/java/fr/osug/doi/repository/DoiStagingRepository.java
+++ b/src/main/java/fr/osug/doi/repository/DoiStagingRepository.java
@@ -3,7 +3,6 @@
******************************************************************************/
package fr.osug.doi.repository;
-import fr.osug.doi.domain.Doi;
import fr.osug.doi.domain.DoiStaging;
import java.util.List;
import org.springframework.data.jpa.repository.JpaRepository;
@@ -17,21 +16,10 @@ import org.springframework.transaction.annotation.Transactional;
*/
@Repository
@Transactional
-public interface DoiStagingRepository extends JpaRepository<DoiStaging, Long> {
+public interface DoiStagingRepository extends DoiBaseRepository<DoiStaging>, JpaRepository<DoiStaging, Long> {
- @Transactional(readOnly = true)
- DoiStaging findOneByDoi(Doi doi);
-
- @Transactional(readOnly = true)
- @Query("select ds from DoiStaging ds join ds.doi d join d.project p where (p.name = :projectName) order by d.identifier asc")
- List<DoiStaging> findByProject(@Param("projectName") String projectName);
-
@Transactional(readOnly = true)
@Query("select ds from DoiStaging ds join ds.doi d join d.project p where (p.name = :projectName) and ds.valid = false order by d.identifier asc")
List<DoiStaging> findErrorByProject(@Param("projectName") String projectName);
- @Transactional(readOnly = true)
- @Query("select count(ds) from DoiStaging ds join ds.doi d join d.project p where (p.name = :projectName)")
- Long countByProject(@Param("projectName") String projectName);
-
}
diff --git a/src/main/java/fr/osug/doi/repository/ProjectRepository.java b/src/main/java/fr/osug/doi/repository/ProjectRepository.java
index 72510ec..77f71f4 100644
--- a/src/main/java/fr/osug/doi/repository/ProjectRepository.java
+++ b/src/main/java/fr/osug/doi/repository/ProjectRepository.java
@@ -6,6 +6,7 @@ package fr.osug.doi.repository;
import fr.osug.doi.domain.Project;
import java.util.List;
import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.data.jpa.repository.Query;
import org.springframework.stereotype.Repository;
import org.springframework.transaction.annotation.Transactional;
@@ -23,4 +24,8 @@ public interface ProjectRepository extends JpaRepository<Project, Long> {
@Transactional(readOnly = true)
List<Project> findAllByOrderByNameAsc();
+ @Transactional(readOnly = true)
+ @Query("select p.name from Project p order by p.name asc")
+ List<String> findAllNamesOrderByNameAsc();
+
}
diff --git a/src/main/java/fr/osug/doi/service/DataciteClient.java b/src/main/java/fr/osug/doi/service/DataciteClient.java
new file mode 100644
index 0000000..1afaac5
--- /dev/null
+++ b/src/main/java/fr/osug/doi/service/DataciteClient.java
@@ -0,0 +1,242 @@
+/*******************************************************************************
+ * OSUG-DOI project ( http://doi.osug.fr ) - Copyright (C) CNRS.
+ ******************************************************************************/
+package fr.osug.doi.service;
+
+import fr.osug.doi.Const;
+import fr.osug.doi.DataciteConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Service;
+import org.springframework.web.client.RestTemplate;
+import org.springframework.boot.web.client.RestTemplateBuilder;
+import org.springframework.http.HttpEntity;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpStatus;
+import org.springframework.web.client.HttpClientErrorException;
+import org.springframework.web.client.RestClientException;
+
+/**
+ *
+ */
+@Service
+public class DataciteClient {
+
+ private final static Logger logger = LoggerFactory.getLogger(DataciteConfig.class.getName());
+
+ public final static boolean RESTRICT_POST_TO_TEST_PREFIX = true;
+
+ public final static int TIMEOUT_CONNECT = 10000;
+ public final static int TIMEOUT_READ = 30000;
+
+ public final static String BASE_URL = "https://mds.datacite.org";
+ /* specific content-types */
+ public final static String PLAIN_UTF8 = "text/plain;charset=UTF-8";
+ public final static String XML_UTF8 = "application/xml;charset=UTF-8";
+
+ private final RestTemplate restTemplate;
+
+ public DataciteClient(RestTemplateBuilder restTemplateBuilder) {
+ this.restTemplate = restTemplateBuilder.rootUri(BASE_URL)
+ .setConnectTimeout(TIMEOUT_CONNECT)
+ .setReadTimeout(TIMEOUT_READ)
+ .build();
+ }
+
+ /**
+ GET (list all DOIs)
+ URI: https://mds.datacite.org/doi
+
+ This request returns a list of all DOIs for the requesting datacentre. There is no guaranteed order.
+
+ Response body
+ If response status is 200: list of DOIs, one DOI per line; empty for 204
+
+ Response statuses
+ 200 OK - operation successful
+ 204 No Content - no DOIs founds
+
+ @return
+ */
+ public String getDois() {
+ String response = null;
+ try {
+ response = this.restTemplate.getForObject("/doi", String.class);
+ } catch (RestClientException rce) {
+ logger.error("getDois: failed", rce);
+ }
+ logger.debug("getDois:\n{}", response);
+ return response;
+ }
+
+ /**
+ Publish a DOI
+
+ @param doi
+ @param url
+ @param metadata
+ @return
+ */
+ public boolean publishDoi(final String doi, final String url, final String metadata) {
+ // 1. Metadata
+ final String oldMetadata = getMetadataForDoi(doi);
+
+ if (oldMetadata == null || !oldMetadata.equals(metadata)) {
+ postMetadataForDoi(doi, metadata);
+ }
+
+ // 2. create / update DOI:
+ return setDoi(doi, url);
+ }
+
+ /**
+ POST
+ URI: https://mds.datacite.org/doi
+
+ POST will mint new DOI if specified DOI doesn't exist.
+ This method will attempt to update URL if you specify existing DOI.
+ A new record in Datasets will be created.
+
+ Request headers
+ Content-Type:text/plain;charset=UTF-8
+
+ Request body
+ doi={doi}
+ url={url}
+ where {doi} and {url} have to be replaced by your DOI and URL, UFT-8 encoded.
+
+ Response body
+ short explanation of status code e.g. CREATED, HANDLE_ALREADY_EXISTS etc
+
+ Response statuses
+ 201 Created - operation successful
+ 400 Bad Request - request body must be exactly two lines: DOI and URL; wrong domain, wrong prefix
+ 401 Unauthorized - no login
+ 403 Forbidden - login problem, quota exceeded
+ 412 Precondition failed - metadata must be uploaded first
+ 500 Internal Server Error - server internal error, try later and if problem persists please contact us
+
+ @param doi
+ @param url
+ @return
+ */
+ public boolean setDoi(final String doi, final String url) {
+ if (RESTRICT_POST_TO_TEST_PREFIX && !doi.startsWith(Const.DOI_PREFIX_TEST)) {
+ throw new IllegalStateException("Only Test prefix is allowed !");
+ }
+ logger.debug("setDoi: doi: {} url: {}", doi, url);
+
+ final HttpHeaders headers = new HttpHeaders();
+ headers.set(HttpHeaders.CONTENT_TYPE, PLAIN_UTF8);
+
+ final String parameters = "doi=" + doi + "\nurl=" + url;
+ logger.debug("parameters: {}", parameters);
+
+ final HttpEntity<String> request = new HttpEntity<String>(parameters, headers);
+
+ boolean result = false;
+ try {
+ final String response = this.restTemplate.postForObject("/doi", request, String.class);
+ logger.debug("setDoi: {}", response);
+ result = true;
+ } catch (RestClientException rce) {
+ logger.error("setDoi: failed", rce);
+ }
+ return result;
+ }
+
+ /* Metadata API */
+ /**
+ GET
+ URI: https://mds.datacite.org/metadata/{doi} where {doi} is a specific DOI.
+
+ This request returns the most recent version of metadata associated with a given DOI.
+
+ Request headers
+ Accept:application/xml
+
+ Response headers
+ Content-Type:application/xml
+
+ Response body
+ If response status is 200: XML representing a dataset, otherwise short explanation for non-200 status
+
+ Response statuses
+ 200 OK - operation successful
+ 401 Unauthorized - no login
+ 403 Forbidden - login problem or dataset belongs to another party
+ 404 Not Found - DOI does not exist in our database
+ 410 Gone - the requested dataset was marked inactive (using DELETE method)
+ 500 Internal Server Error - server internal error, try later and if problem persists please contact us
+
+ @param doi
+ @return
+ */
+ public String getMetadataForDoi(final String doi) {
+ String response = null;
+ try {
+ response = this.restTemplate.getForObject("/metadata/{doi}", String.class, doi);
+ } catch (HttpClientErrorException hcee) {
+ if (hcee.getStatusCode() == HttpStatus.NOT_FOUND) {
+ logger.debug("getMetadataForDoi: failed", hcee);
+ } else {
+ logger.error("getMetadataForDoi: failed", hcee);
+ }
+ } catch (RestClientException rce) {
+ logger.error("getMetadataForDoi: failed", rce);
+ }
+ logger.debug("getMetadataForDoi:\n{}", response);
+ return response;
+ }
+
+ /**
+ POST
+ URI: https://mds.datacite.org/metadata
+
+ This request stores new version of metadata. The request body must contain valid XML.
+
+ Request headers
+ Content-Type:application/xml;charset=UTF-8
+
+ Request body
+ UFT-8 encoded metadata
+
+ Response body
+ short explanation of status code e.g. CREATED, HANDLE_ALREADY_EXISTS etc
+
+ Response headers
+ Location - URL of the newly stored metadata
+
+ Response statuses
+ 201 Created - operation successful
+ 400 Bad Request - invalid XML, wrong prefix
+ 401 Unauthorized - no login
+ 403 Forbidden - login problem, quota exceeded
+ 500 Internal Server Error - server internal error, try later and if problem persists please contact us
+
+ @param doi
+ @param metadata
+ @return
+ */
+ public boolean postMetadataForDoi(final String doi, final String metadata) {
+ if (RESTRICT_POST_TO_TEST_PREFIX && !doi.startsWith(Const.DOI_PREFIX_TEST)) {
+ throw new IllegalStateException("Only Test prefix is allowed !");
+ }
+ logger.debug("postMetadataForDoi: metadata: {}", metadata);
+
+ final HttpHeaders headers = new HttpHeaders();
+ headers.set(HttpHeaders.CONTENT_TYPE, XML_UTF8);
+
+ final HttpEntity<String> request = new HttpEntity<String>(metadata, headers);
+
+ boolean result = false;
+ try {
+ final String response = this.restTemplate.postForObject("/metadata", request, String.class);
+ logger.debug("postMetadataForDoi: {}", response);
+ result = true;
+ } catch (RestClientException rce) {
+ logger.error("postMetadataForDoi: failed", rce);
+ }
+ return result;
+ }
+}
diff --git a/src/main/java/fr/osug/doi/service/DoiService.java b/src/main/java/fr/osug/doi/service/DoiService.java
index 7654513..c393c35 100644
--- a/src/main/java/fr/osug/doi/service/DoiService.java
+++ b/src/main/java/fr/osug/doi/service/DoiService.java
@@ -3,15 +3,18 @@
******************************************************************************/
package fr.osug.doi.service;
+import fr.osug.doi.ProcessPipelineDoiData;
import fr.osug.doi.domain.Doi;
import fr.osug.doi.domain.DoiPublic;
import fr.osug.doi.domain.DoiStaging;
import fr.osug.doi.domain.Project;
import fr.osug.doi.domain.Status;
+import fr.osug.doi.repository.DoiBaseRepository;
import fr.osug.doi.repository.DoiPublicRepository;
import fr.osug.doi.repository.DoiRepository;
import fr.osug.doi.repository.DoiStagingRepository;
import fr.osug.doi.repository.ProjectRepository;
+import java.util.Collection;
import java.util.Date;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -25,7 +28,7 @@ import org.springframework.transaction.annotation.Transactional;
@Service
public class DoiService {
- private final Logger log = LoggerFactory.getLogger(DoiService.class);
+ private final static Logger logger = LoggerFactory.getLogger(DoiService.class);
@Autowired
private ProjectRepository projectRepository;
@@ -39,6 +42,28 @@ public class DoiService {
@Autowired
private DoiPublicRepository doiPublicRepository;
+ @Transactional
+ public void importDoiStagingCollection(final String projectName, final Date now,
+ final Collection<ProcessPipelineDoiData> doiDataCollection) {
+
+ // Get or create the project:
+ final Project project = getOrCreateProject(projectName, now);
+
+ // Update Dois:
+ for (ProcessPipelineDoiData doiData : doiDataCollection) {
+ // Update DOI in staging phase:
+ createOrUpdateStagingDoi(project,
+ doiData.getDoiSuffix(), doiData.getTitle(),
+ doiData.getDataAccessUrl(), doiData.getLandingPageExtUrl(),
+ doiData.isValid(), doiData.getMetadataMd5(),
+ doiData.messagesToString(), now,
+ doiData.getLandingLocUrl());
+ }
+
+ // Update Project:
+ updateProjectDate(projectName, now);
+ }
+
@Transactional
public Project getOrCreateProject(final String name, final Date now) {
Project project = projectRepository.findOneByName(name);
@@ -50,7 +75,7 @@ public class DoiService {
project.setCreateDate(now);
project.setUpdateDate(now);
- log.debug("saving: {}", project);
+ logger.debug("saving: {}", project);
projectRepository.save(project);
}
@@ -67,7 +92,7 @@ public class DoiService {
project.setUpdateDate(now);
- log.debug("saving: {}", project);
+ logger.debug("saving: {}", project);
projectRepository.save(project);
}
@@ -79,13 +104,8 @@ public class DoiService {
@Transactional
public Doi createOrUpdateDoi(final Project project, final String identifier,
- final String description, final String landingExtUrl) {
- return createOrUpdateDoi(project, identifier, Status.STAGING, description, landingExtUrl);
- }
-
- @Transactional
- public Doi createOrUpdateDoi(final Project project, final String identifier,
- final Status status, final String description, final String landingExtUrl) {
+ final Status status, final String description,
+ final String dataAccessUrl, final String landingExtUrl) {
Doi doi = doiRepository.findOneByIdentifier(identifier);
@@ -95,11 +115,15 @@ public class DoiService {
doi.setIdentifier(identifier);
}
- doi.setStatus(status);
+ // only update status (PUBLIC > STAGING)
+ if (doi.getStatus() != Status.PUBLIC) {
+ doi.setStatus(status);
+ }
doi.setDescription(description);
+ doi.setDataAccessUrl(dataAccessUrl);
doi.setLandingExtUrl(landingExtUrl);
- log.debug("saving: {}", doi);
+ logger.debug("saving: {}", doi);
doiRepository.save(doi);
@@ -108,12 +132,13 @@ public class DoiService {
@Transactional
public DoiStaging createOrUpdateStagingDoi(final Project project, final String identifier,
- final String description, final String landingExtUrl,
+ final String description,
+ final String dataAccessUrl, final String landingExtUrl,
final boolean valid, final String metadataMd5,
final String processLog,
- final Date now) {
+ final Date now, final String landingLocUrl) {
- final Doi doi = createOrUpdateDoi(project, identifier, description, landingExtUrl);
+ final Doi doi = createOrUpdateDoi(project, identifier, Status.STAGING, description, dataAccessUrl, landingExtUrl);
DoiStaging doiStaging = doiStagingRepository.findOneByDoi(doi);
@@ -126,17 +151,17 @@ public class DoiService {
} else {
// update date only if the MD5 changed:
if (metadataMd5 != null && !metadataMd5.equals(doiStaging.getMetadataMd5())) {
- log.info("{} md5 modified: {}", doi.getIdentifier(), metadataMd5);
+ logger.info("{} md5 modified: {}", doi.getIdentifier(), metadataMd5);
doiStaging.setUpdateDate(now);
}
}
doiStaging.setValid(valid);
doiStaging.setMetadataMd5(metadataMd5);
- doiStaging.setLandingLocUrl("http://doi.osug.fr/staging/" + doi.getIdentifier()); // TODO
+ doiStaging.setLandingLocUrl(landingLocUrl);
doiStaging.setLog(processLog);
- log.debug("saving: {}", doiStaging);
+ logger.debug("saving: {}", doiStaging);
doiStagingRepository.save(doiStaging);
@@ -145,7 +170,7 @@ public class DoiService {
@Transactional
public DoiPublic createOrUpdatePublicDoi(final Project project, final DoiStaging doiStaging,
- final Date now) {
+ final Date now, final String landingLocUrl) {
final Doi doi = doiStaging.getDoi();
DoiPublic doiPublic = doiPublicRepository.findOneByDoi(doi);
@@ -158,11 +183,11 @@ public class DoiService {
}
doiPublic.setMetadataMd5(doiStaging.getMetadataMd5());
- doiPublic.setLandingLocUrl("http://doi.osug.fr/public/" + doi.getIdentifier()); // TODO
- doiPublic.setActiveExtUrl(false); // TODO
+ doiPublic.setLandingLocUrl(landingLocUrl);
+ doiPublic.setActiveExtUrl(true); // TODO: handle external url checks
doiPublic.setUpdateDate(now);
- log.debug("saving: {}", doiPublic);
+ logger.debug("saving: {}", doiPublic);
doiPublicRepository.save(doiPublic);
@@ -177,12 +202,16 @@ public class DoiService {
return doiRepository;
}
- public DoiStagingRepository getDoiStagingRepository() {
- return doiStagingRepository;
+ public DoiBaseRepository<?> getDoiCommonRepository(final boolean isStaging) {
+ return (isStaging) ? doiStagingRepository : doiPublicRepository;
}
public DoiPublicRepository getDoiPublicRepository() {
return doiPublicRepository;
}
+ public DoiStagingRepository getDoiStagingRepository() {
+ return doiStagingRepository;
+ }
+
}
diff --git a/src/main/java/fr/osug/doi/validation/ValidationUtil.java b/src/main/java/fr/osug/doi/validation/ValidationUtil.java
index c715595..d6852f6 100644
--- a/src/main/java/fr/osug/doi/validation/ValidationUtil.java
+++ b/src/main/java/fr/osug/doi/validation/ValidationUtil.java
@@ -4,21 +4,38 @@
package fr.osug.doi.validation;
import fr.osug.doi.Const;
-import fr.osug.doi.PipelineDoiData;
+import fr.osug.doi.ProcessPipelineDoiData;
import fr.osug.doi.DoiCsvData;
+import java.text.Normalizer;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Pattern;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
*
- * @author bourgesl
*/
public class ValidationUtil {
private final static Logger logger = LoggerFactory.getLogger(ValidationUtil.class.getName());
+ /** Empty String constant '' */
+ public final static String STRING_EMPTY = "";
+ /** String constant containing 1 space character ' ' */
+ public final static String STRING_SPACE = " ";
+
/** regular expression used to match characters different than alpha/numeric/_/-/. (1..n) */
private final static Pattern PATTERN_IDENTIFIER = Pattern.compile("[^a-zA-Z0-9\\-_\\.]");
+ /** regular expression used to match characters different than alpha (1..n) */
+ private final static Pattern PATTERN_NON_ALPHA = Pattern.compile("[^a-zA-Z]+");
+
+ /** regular expression used to match characters with accents */
+ private final static Pattern PATTERN_ACCENT_CHARS = Pattern.compile("\\p{InCombiningDiacriticalMarks}+");
+
+ /** RegExp expression to match white spaces (1..n) */
+ private final static Pattern PATTERN_WHITE_SPACE_MULTIPLE = Pattern.compile("\\s+");
+ /** simplify name cache */
+ private final static Map<String, String> cacheSimpleNames = new ConcurrentHashMap<String, String>(128);
private ValidationUtil() {
}
@@ -40,7 +57,7 @@ public class ValidationUtil {
return true;
}
- public static void validateIdentifier(final String id, final PipelineDoiData pipeData) {
+ public static void validateIdentifier(final String id, final ProcessPipelineDoiData pipeData) {
// Check test prefix:
if (!isTestPrefix(id)) {
// identifier does not starts with the test prefix
@@ -53,7 +70,7 @@ public class ValidationUtil {
pipeData.addError("Invalid suffix for ''" + Const.KEY_IDENTIFIER + "'' = '" + suffix + "'");
}
}
-
+
public static boolean isTestPrefix(final String id) {
return id.startsWith(Const.DOI_PREFIX_TEST);
}
@@ -66,4 +83,72 @@ public class ValidationUtil {
private static boolean checkIdentifier(final String value) {
return !PATTERN_IDENTIFIER.matcher(value).matches();
}
+
+ /** string utils */
+ /**
+ * Test if value is empty (null or no chars)
+ *
+ * @param value string value
+ * @return true if value is empty (null or no chars)
+ */
+ public static boolean isEmpty(final String value) {
+ return value == null || value.length() == 0;
+ }
+
+ /**
+ * Replace any non alpha numeric character by ' ' and remove accents
+ * @param value input value
+ * @return string value
+ */
+ public static String simplifyName(final String value) {
+ String res = cacheSimpleNames.get(value);
+ if (res == null) {
+ res = cleanWhiteSpaces(replaceNonAlphaChars(removeAccents(value), STRING_SPACE));
+ logger.debug("simplifyName: '{}' => '{}'", value, res);
+ cacheSimpleNames.put(value, res);
+ }
+ return res;
+ }
+
+ /**
+ * Replace non alpha numeric characters by the given replacement string
+ * @param value input value
+ * @param replaceBy replacement string
+ * @return string value
+ */
+ private static String replaceNonAlphaChars(final String value, final String replaceBy) {
+ return PATTERN_NON_ALPHA.matcher(value).replaceAll(replaceBy);
+ }
+
+ /**
+ * Remove accents from any character i.e. remove diacritical marks
+ * @param value input value
+ * @return string value
+ */
+ private static String removeAccents(final String value) {
+ // Remove accent from characters (if any) (Java 1.6)
+ final String normalized = Normalizer.normalize(value, Normalizer.Form.NFD);
+
+ return PATTERN_ACCENT_CHARS.matcher(normalized).replaceAll(STRING_EMPTY);
+ }
+
+ /**
+ * Trim and remove redundant white space characters
+ * @param value input value
+ * @return string value
+ */
+ private static String cleanWhiteSpaces(final String value) {
+ return isEmpty(value) ? STRING_EMPTY : replaceWhiteSpaces(value.trim(), STRING_SPACE);
+ }
+
+ /**
+ * Replace white space characters (1..n) by the given replacement string
+ * @param value input value
+ * @param replaceBy replacement string
+ * @return string value
+ */
+ private static String replaceWhiteSpaces(final String value, final String replaceBy) {
+ return PATTERN_WHITE_SPACE_MULTIPLE.matcher(value).replaceAll(replaceBy);
+ }
+
}
diff --git a/src/main/java/fr/osug/doi/validation/WarningMessage.java b/src/main/java/fr/osug/doi/validation/WarningMessage.java
index 2920418..7ef48b8 100644
--- a/src/main/java/fr/osug/doi/validation/WarningMessage.java
+++ b/src/main/java/fr/osug/doi/validation/WarningMessage.java
@@ -5,7 +5,6 @@ package fr.osug.doi.validation;
/**
* This class represents a warning message (message + state)
- * @author bourgesl
*/
public final class WarningMessage {
diff --git a/src/main/java/fr/osug/util/FileUtils.java b/src/main/java/fr/osug/util/FileUtils.java
index 9940ee1..db8ec09 100644
--- a/src/main/java/fr/osug/util/FileUtils.java
+++ b/src/main/java/fr/osug/util/FileUtils.java
@@ -16,9 +16,11 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
+import java.io.OutputStreamWriter;
import java.io.Reader;
import java.io.Writer;
import java.net.URL;
+import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
@@ -355,6 +357,32 @@ public final class FileUtils {
return null;
}
+ public static Reader getTextReader(final File file) throws IOException {
+ return new InputStreamReader(new FileInputStream(file), StandardCharsets.UTF_8);
+ }
+
+ public static Writer getTextWriter(final File file) throws IOException {
+ return getTextWriter(new FileOutputStream(file));
+ }
+
+ public static Writer getTextWriter(final OutputStream out) throws IOException {
+ return new OutputStreamWriter(out, StandardCharsets.UTF_8);
+ }
+
+ public static void writeFile(final String content, final File textFile) throws IOException {
+ log.info("Writing: {}", textFile.getAbsolutePath());
+
+ BufferedWriter writer = null;
+ try {
+ writer = new BufferedWriter(getTextWriter(textFile), content.length());
+ writer.write(content);
+ } finally {
+ if (writer != null) {
+ writer.close();
+ }
+ }
+ }
+
/**
* Read a text file from the given file
*
@@ -470,19 +498,6 @@ public final class FileUtils {
}
}
- public static long computeChecksum(final InputStream in) throws IOException {
-
- final ChecksumOutputStream cs = new ChecksumOutputStream();
-
- final byte[] buf = new byte[DEFAULT_READ_BUFFER_SIZE];
-
- int len;
- while ((len = in.read(buf)) > 0) {
- cs.write(buf, 0, len);
- }
- return cs.getChecksum(); // flush and close streams
- }
-
public static String MD5(final InputStream in) throws IOException {
try {
final MessageDigest md = MessageDigest.getInstance("MD5");
diff --git a/src/main/java/fr/osug/xml/XmlFactory.java b/src/main/java/fr/osug/xml/XmlFactory.java
index 7a74c9b..f64de2f 100644
--- a/src/main/java/fr/osug/xml/XmlFactory.java
+++ b/src/main/java/fr/osug/xml/XmlFactory.java
@@ -32,7 +32,6 @@ import javax.xml.validation.Schema;
import javax.xml.validation.SchemaFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
@@ -95,7 +94,7 @@ public final class XmlFactory {
try {
transformerFactory = TransformerFactory.newInstance();
- log.info("transformerFactory: {}", transformerFactory);
+ log.debug("transformerFactory: {}", transformerFactory);
} catch (final TransformerFactoryConfigurationError tfce) {
throw new IllegalStateException("XmlFactory.getTransformerFactory : failure on TransformerFactory initialisation : ", tfce);
@@ -385,6 +384,9 @@ public final class XmlFactory {
if (tf != null) {
if (parameters != null) {
+ if (log.isDebugEnabled()) {
+ log.debug("XmlFactory.transform : XSL parameters : {}", parameters);
+ }
for (final Map.Entry<String, Object> entry : parameters.entrySet()) {
tf.setParameter(entry.getKey(), entry.getValue());
}
diff --git a/src/main/resources/config/application-default.yml b/src/main/resources/config/application-default.yml
index 998c874..944b99b 100644
--- a/src/main/resources/config/application-default.yml
+++ b/src/main/resources/config/application-default.yml
@@ -15,17 +15,21 @@ debug: true
spring:
datasource:
# go into osug-doi/tmp/ folder:
- url: jdbc:h2:file:../tmp/h2db/doi;MODE=PostgreSQL;DB_CLOSE_DELAY=-1;DB_CLOSE_ON_EXIT=FALSE
- name:
+ url: jdbc:h2:../tmp/h2db/doi;MODE=PostgreSQL
+#;DB_CLOSE_DELAY=-1;DB_CLOSE_ON_EXIT=FALSE
+ name: doi
username: doi
password:
+ initial-size: 1
+ min-idle: 2
+ max-idle: 4
h2:
console:
- enabled: true
+ enabled: false
jpa:
database: H2
database-platform: org.hibernate.dialect.PostgreSQL9Dialect
show_sql: true
- hibernate:
- ddl-auto: none
+# hibernate:
+# ddl-auto: none
# ddl-auto: create
diff --git a/src/main/resources/config/application-test.yml b/src/main/resources/config/application-test.yml
index 998c874..d2be6b0 100644
--- a/src/main/resources/config/application-test.yml
+++ b/src/main/resources/config/application-test.yml
@@ -14,18 +14,17 @@ debug: true
spring:
datasource:
-# go into osug-doi/tmp/ folder:
- url: jdbc:h2:file:../tmp/h2db/doi;MODE=PostgreSQL;DB_CLOSE_DELAY=-1;DB_CLOSE_ON_EXIT=FALSE
- name:
+ url: jdbc:h2:mem:doi;MODE=PostgreSQL;DB_CLOSE_DELAY=-1;DB_CLOSE_ON_EXIT=FALSE
+ name: doi
username: doi
password:
+ initial-size: 1
+ min-idle: 2
+ max-idle: 4
h2:
console:
- enabled: true
+ enabled: false
jpa:
database: H2
database-platform: org.hibernate.dialect.PostgreSQL9Dialect
show_sql: true
- hibernate:
- ddl-auto: none
-# ddl-auto: create
diff --git a/src/main/resources/config/application.yml b/src/main/resources/config/application.yml
index 6280ae6..e2a727f 100644
--- a/src/main/resources/config/application.yml
+++ b/src/main/resources/config/application.yml
@@ -24,11 +24,13 @@ spring:
jpa:
open-in-view: false
properties:
- hibernate.id.new_generator_mappings: false
+ hibernate.id.new_generator_mappings: true
hibernate.cache.use_second_level_cache: false
hibernate.cache.use_query_cache: false
hibernate.generate_statistics: false
- hibernate.hbm2ddl.auto: none
+ hibernate.hbm2ddl.auto:
+ main:
+ web-environment: false
logging:
file: osug-doi.log
@@ -36,4 +38,10 @@ logging:
osug:
doi:
prefix: 10.5072
- domain: my-domain.fr
+ domain: http://my-domain.fr/
+ dataciteClientEnabled: false
+
+datacite:
+ user: user
+ password: password_to_define
+
diff --git a/src/main/resources/db/migration/V1__init.sql b/src/main/resources/db/migration/V1__init.sql
index 8f52947..cbdfce5 100644
--- a/src/main/resources/db/migration/V1__init.sql
+++ b/src/main/resources/db/migration/V1__init.sql
@@ -1,6 +1,6 @@
/* PROJECT */
-create sequence project_id_seq START 1;
+create sequence project_id_seq START 1 INCREMENT 10;
create table project (
id int8 not null,
@@ -14,7 +14,7 @@ create table project (
alter table project add constraint uk_project_name unique (name);
/* DOI */
-create sequence doi_id_seq START 1;
+create sequence doi_id_seq START 1 INCREMENT 10;
create table doi (
id int8 not null,
@@ -22,39 +22,43 @@ create table doi (
identifier varchar(255) not null,
status varchar(16),
description varchar(512),
+ data_access_url varchar(1024),
landing_ext_url varchar(1024),
primary key (id)
);
alter table doi add constraint uk_doi_identifier unique (identifier);
alter table doi add constraint fk_doi_project_id foreign key (project_id) references project;
-/* DOI STAGING */
-create sequence doi_staging_id_seq START 1;
+/* DOI COMMON (seq) */
+create sequence doi_common_id_seq START 1 INCREMENT 10;
+/* DOI STAGING */
create table doi_staging (
+ /* DOI Common fields */
id int8 not null,
doi_id int8 not null,
- valid boolean,
metadata_md5 varchar(32) not null,
- log varchar(4096),
landing_loc_url varchar(512) not null,
create_date timestamp,
update_date timestamp,
+ /* specific fields */
+ valid boolean,
+ log varchar(4096),
primary key (id)
);
alter table doi_staging add constraint fk_doi_staging_id foreign key (doi_id) references doi;
/* DOI PUBLIC */
-create sequence doi_public_id_seq START 1;
-
create table doi_public (
+ /* DOI Common fields */
id int8 not null,
doi_id int8 not null,
metadata_md5 varchar(32) not null,
landing_loc_url varchar(512) not null,
- active_ext_url boolean,
create_date timestamp,
update_date timestamp,
+ /* specific fields */
+ active_ext_url boolean,
primary key (id)
);
alter table doi_public add constraint fk_doi_public_id foreign key (doi_id) references doi;
diff --git a/src/main/resources/logback-spring.xml b/src/main/resources/logback-spring.xml
index fdd305a..a496fae 100644
--- a/src/main/resources/logback-spring.xml
+++ b/src/main/resources/logback-spring.xml
@@ -8,8 +8,11 @@
<logger name="fr.osug.doi" level="DEBUG" />
<logger name="org.springframework.orm.jpa.JpaTransactionManager" level="DEBUG"/>
- <logger name="org.hibernate.transaction" level="DEBUG" />
<logger name="org.springframework.transaction" level="DEBUG" />
+ <logger name="org.springframework.transaction.interceptor" level="TRACE" />
+
+ <logger name="org.springframework.web.client.RestTemplate" level="TRACE" />
+
</springProfile>
<springProfile name="prod">
diff --git a/src/test/java/fr/osug/doi/DOIServiceTest.java b/src/test/java/fr/osug/doi/DOIServiceTest.java
index e72cc51..c8f5708 100644
--- a/src/test/java/fr/osug/doi/DOIServiceTest.java
+++ b/src/test/java/fr/osug/doi/DOIServiceTest.java
@@ -27,11 +27,10 @@ import org.springframework.transaction.annotation.Transactional;
/**
*
- * @author bourgesl
*/
@RunWith(SpringJUnit4ClassRunner.class)
@SpringBootTest
-@ActiveProfiles("test,default")
+@ActiveProfiles({"test","debug"})
public class DOIServiceTest {
private final static Logger logger = LoggerFactory.getLogger(DOIServiceTest.class.getName());
@@ -50,14 +49,14 @@ public class DOIServiceTest {
}
final ProjectRepository pr = doiService.getProjectRepository();
- /*
- final Sort sortByName = new Sort("name");
- final List<Project> projects = pr.findAll(sortByName);
- */
+
+ final List<String> names = pr.findAllNamesOrderByNameAsc();
+ logger.info("names: {}", names);
+
final List<Project> projects = pr.findAllByOrderByNameAsc();
for (Project dp : projects) {
- logger.info("remove project: " + dp);
+ logger.info("remove project: {}", dp);
pr.delete(dp);
}
@@ -91,7 +90,7 @@ public class DOIServiceTest {
final List<Project> projects = pr.findAllByOrderByNameAsc();
for (Project dp : projects) {
- logger.info("remove project: " + dp);
+ logger.info("remove project: {}", dp);
pr.delete(dp);
}
logger.info("verifyInitialDatabaseState: exit");
@@ -106,7 +105,7 @@ public class DOIServiceTest {
final List<Project> projects = pr.findAllByOrderByNameAsc();
for (Project dp : projects) {
- logger.info("project: " + dp);
+ logger.info("project: {}", dp);
}
Assert.assertTrue(projects.isEmpty());
@@ -118,7 +117,7 @@ public class DOIServiceTest {
@Test
public void testCreateProjectAndDoi() throws Exception {
final String projectName = "AMMA-CATCH";
-
+
final Date now = new Date();
final Project project = doiService.getOrCreateProject(projectName, now);
@@ -126,9 +125,10 @@ public class DOIServiceTest {
for (int i = 0; i < 93; i++) {
Assert.assertNotNull(doiService.createOrUpdateStagingDoi(project,
- "AMMA-CATCH.TEST." + (int) (10000.0 * Math.random()),
- "description ...", "http://server/ext",
- true, "md5", "log:blah\nblah", now));
+ "AMMA-CATCH.TEST." + (int) (10.0 * Math.random()),
+ "description ...", "http://server/data/file", "http://server/ext/page",
+ true, "md5", "log:blah\nblah", now,
+ "/test/AMMA-CATCH.TEST.html"));
Thread.sleep(20l);
}
@@ -142,7 +142,7 @@ public class DOIServiceTest {
logger.info("Dois for project[{}]: {}", projectName, dois.size());
for (Doi d : dois) {
- logger.info("Doi: " + d);
+ logger.info("Doi: {}", d);
}
final DoiStagingRepository dsr = doiService.getDoiStagingRepository();
@@ -151,7 +151,7 @@ public class DOIServiceTest {
logger.info("DoiStaging for project[{}]: {}", projectName, dois.size());
for (DoiStaging d : doiStagings) {
- logger.info("DoiStaging: " + d);
+ logger.info("DoiStaging: {}", d);
}
}
}
diff --git a/src/test/java/fr/osug/doi/DOITextToXmlTest.java b/src/test/java/fr/osug/doi/DOITextToXmlTest.java
index ad07676..205dc8f 100644
--- a/src/test/java/fr/osug/doi/DOITextToXmlTest.java
+++ b/src/test/java/fr/osug/doi/DOITextToXmlTest.java
@@ -26,11 +26,10 @@ import org.springframework.transaction.annotation.Transactional;
/**
*
- * @author bourgesl
*/
@RunWith(SpringJUnit4ClassRunner.class)
@SpringBootTest
-@ActiveProfiles(profiles = {"test", "debug"})
+@ActiveProfiles(profiles = {"test","debug"})
public class DOITextToXmlTest {
private final static Logger logger = LoggerFactory.getLogger(DOITextToXmlTest.class.getName());
@@ -43,9 +42,7 @@ public class DOITextToXmlTest {
private final static String DIR_OUTPUT = Paths.DIR_BASE + "target/test-output/";
private final static String DIR_STAGING = DIR_OUTPUT + "staging/";
public final static String DIR_WEB = DIR_OUTPUT + "www/";
- public final static String DIR_WEB_STAGING = DIR_WEB + "staging/";
- public final static String DIR_WEB_PUBLIC = DIR_WEB + "public/";
-
+
@BeforeClass
public static void setUpClass() {
}
@@ -84,22 +81,21 @@ public class DOITextToXmlTest {
@Transactional
@Test
public void testPipeline() throws IOException {
-
+
final String project = "AMMA-CATCH";
- ProjectConfig pc = new ProjectConfig(project);
+ final PathConfig pathConfig = doiConfig.getPathConfig();
+ // redirect web:
+ pathConfig.initWebRoot(DIR_WEB);
+
+ final ProjectConfig pc = new ProjectConfig(pathConfig, project);
pc.setSaveCSV(true);
+ // redirect outputs:
pc.initInputDir(DIR_TEST);
pc.initTmpDir(DIR_OUTPUT);
pc.initStagingDir(DIR_STAGING);
- pc.initWebStagingDir(DIR_WEB_STAGING);
- pc.initWebStagingProjectDir(DIR_WEB_STAGING + project);
- pc.initWebStagingProjectEmbedDir(DIR_WEB_STAGING + project + '/' + Paths.DIR_WEB_EMBED);
- pc.initWebStagingProjectXmlDir(DIR_WEB_STAGING + project + '/' + Paths.DIR_WEB_XML);
- pc.initWebPublicDir(DIR_WEB_PUBLIC);
- pc.initWebPublicProjectDir(DIR_WEB_PUBLIC + project);
final ProcessPipeline pipeline = new ProcessPipeline(doiConfig, pc);
pipeline.process();
--
GitLab