Skip to content

GitLab

S
scidetect
Commit 03da7eb9 authored by Tien's avatar Tien
Browse files
Options

Updated readme, nbproject

parent f849cbfa
Hide whitespace changes
Inline Side-by-side
Showing with 738 additions and 61 deletions
Makefile
View file @ 03da7eb9
......@@ -15,17 +15,17 @@ all: classes doc jar run
classes:
mkdir -p classes
$(JAVAC) $(JAVAFLAGS) $(JAVACLASSPATH) src/fr/imag/forge/Scidetect/*/*.java
$(JAVAC) $(JAVAFLAGS) $(JAVACLASSPATH) src/fr/imag/forge/scidetect/*/*.java
doc:
$(JAVADOC) src/*/*/*/*/*/*
jar:
cd classes ; jar -cfvm ../ScigenChecker_Local`date +%Y-%m-%d`.jar ../manifest.mf *; cd ..
cp ScigenChecker_Local`date +%Y-%m-%d`.jar ScigenChecker_Local.jar
cd classes ; jar -cfvm ../SciDetect_Local`date +%Y-%m-%d`.jar ../manifest.mf *; cd ..
cp SciDetect_Local`date +%Y-%m-%d`.jar SciDetect_Local.jar
run:
java -jar ScigenChecker_local.jar -l checklog.txt -c Test
java -jar SciDetect_local.jar -l checklog.txt -c Test
clean:
rm -r classes; rm -r doc;
dist/README.TXT deleted 100644 → 0
View file @ f849cbfa
========================
BUILD OUTPUT DESCRIPTION
========================
When you build an Java application project that has a main class, the IDE
automatically copies all of the JAR
files on the projects classpath to your projects dist/lib folder. The IDE
also adds each of the JAR files to the Class-Path element in the application
JAR files manifest file (MANIFEST.MF).
To run the project from the command line, go to the dist folder and
type the following:
java -jar "ScigenChecker_Local.jar"
To distribute this project, zip up the dist folder (including the lib folder)
and distribute the ZIP file.
Notes:
* If two JAR files on the project classpath have the same name, only the first
JAR file is copied to the lib folder.
* Only JAR files are copied to the lib folder.
If the classpath contains other types of files or folders, these files (folders)
are not copied.
* If a library on the projects classpath also has a Class-Path element
specified in the manifest,the content of the Class-Path element has to be on
the projects runtime path.
* To set a main class in a standard Java project, right-click the project node
in the Projects window and choose Properties. Then click Run and enter the
class name in the Main Class field. Alternatively, you can manually type the
class name in the manifest Main-Class element.
nbproject/build-impl.xml
View file @ 03da7eb9
......@@ -19,7 +19,7 @@ is divided into following sections:
- cleanup
-->
<project xmlns:j2seproject1="http://www.netbeans.org/ns/j2se-project/1" xmlns:j2seproject3="http://www.netbeans.org/ns/j2se-project/3" xmlns:jaxrpc="http://www.netbeans.org/ns/j2se-project/jax-rpc" basedir=".." default="default" name="ScigenChecker_Local-impl">
<project xmlns:j2seproject1="http://www.netbeans.org/ns/j2se-project/1" xmlns:j2seproject3="http://www.netbeans.org/ns/j2se-project/3" xmlns:jaxrpc="http://www.netbeans.org/ns/j2se-project/jax-rpc" basedir=".." default="default" name="SciDetect_Local-impl">
<fail message="Please build using Ant 1.8.0 or higher.">
<condition>
<not>
......@@ -470,7 +470,7 @@ is divided into following sections:
</fileset>
</union>
<taskdef classname="org.testng.TestNGAntTask" classpath="${run.test.classpath}" name="testng"/>
<testng classfilesetref="test.set" failureProperty="tests.failed" jvm="${platform.java}" listeners="org.testng.reporters.VerboseReporter" methods="${testng.methods.arg}" mode="${testng.mode}" outputdir="${build.test.results.dir}" suitename="ScigenChecker_Local" testname="TestNG tests" workingDir="${work.dir}">
<testng classfilesetref="test.set" failureProperty="tests.failed" jvm="${platform.java}" listeners="org.testng.reporters.VerboseReporter" methods="${testng.methods.arg}" mode="${testng.mode}" outputdir="${build.test.results.dir}" suitename="SciDetect_Local" testname="TestNG tests" workingDir="${work.dir}">
<xmlfileset dir="${build.test.classes.dir}" includes="@{testincludes}"/>
<propertyset>
<propertyref prefix="test-sys-prop."/>
......@@ -621,7 +621,7 @@ is divided into following sections:
<condition else="-testclass @{testClass}" property="test.class.or.method" value="-methods @{testClass}.@{testMethod}">
<isset property="test.method"/>
</condition>
<condition else="-suitename ScigenChecker_Local -testname @{testClass} ${test.class.or.method}" property="testng.cmd.args" value="@{testClass}">
<condition else="-suitename SciDetect_Local -testname @{testClass} ${test.class.or.method}" property="testng.cmd.args" value="@{testClass}">
<matches pattern=".*\.xml" string="@{testClass}"/>
</condition>
<delete dir="${build.test.results.dir}" quiet="true"/>
......@@ -918,7 +918,7 @@ is divided into following sections:
<delete file="${built-jar.properties}" quiet="true"/>
</target>
<target if="already.built.jar.${basedir}" name="-warn-already-built-jar">
<echo level="warn" message="Cycle detected: ScigenChecker_Local was already built"/>
<echo level="warn" message="Cycle detected: SciDetect_Local was already built"/>
</target>
<target depends="init,-deps-jar-init" name="deps-jar" unless="no.deps">
<mkdir dir="${build.dir}"/>
......@@ -1406,7 +1406,7 @@ is divided into following sections:
<delete file="${built-clean.properties}" quiet="true"/>
</target>
<target if="already.built.clean.${basedir}" name="-warn-already-built-clean">
<echo level="warn" message="Cycle detected: ScigenChecker_Local was already built"/>
<echo level="warn" message="Cycle detected: SciDetect_Local was already built"/>
</target>
<target depends="init,-deps-clean-init" name="deps-clean" unless="no.deps">
<mkdir dir="${build.dir}"/>
......
nbproject/genfiles.properties
View file @ 03da7eb9
build.xml.data.CRC32=ab46f703
build.xml.script.CRC32=1904251a
build.xml.data.CRC32=97495bee
build.xml.script.CRC32=8bc49660
build.xml.stylesheet.CRC32=8064a381@1.75.2.48
# This file is used by a NetBeans-based IDE to track changes in generated files such as build-impl.xml.
# Do not edit this file. You may delete it but then the IDE will never regenerate such files for you.
nbproject/build-impl.xml.data.CRC32=ab46f703
nbproject/build-impl.xml.script.CRC32=fb0b4f40
nbproject/build-impl.xml.data.CRC32=97495bee
nbproject/build-impl.xml.script.CRC32=850831d4
nbproject/build-impl.xml.stylesheet.CRC32=876e7a8f@1.75.2.48
nbproject/private/private.properties
View file @ 03da7eb9
application.args=-c /home/tien/Test_demo/978-3-642-33030-8_Chapter_10.pdf.xtx
application.args=-c /home/tien/Test_demo -d
compile.on.save=true
do.depend=false
do.jar=true
......
nbproject/private/private.xml
View file @ 03da7eb9
......@@ -2,6 +2,10 @@
<project-private xmlns="http://www.netbeans.org/ns/project-private/1">
<editor-bookmarks xmlns="http://www.netbeans.org/ns/editor-bookmarks/2" lastBookmarkId="0"/>
<open-files xmlns="http://www.netbeans.org/ns/projectui-open-files/2">
<group/>
<group>
<file>file:/home/tien/NetBeansProjects/ScigenChecker_Local/src/fr/imag/forge/scidetect/Checker/Utils/DistancesSet.java</file>
<file>file:/home/tien/NetBeansProjects/ScigenChecker_Local/src/fr/imag/forge/scidetect/Logger/Log.java</file>
<file>file:/home/tien/NetBeansProjects/ScigenChecker_Local/src/fr/imag/forge/scidetect/scigenchecker_local/ScigenChecker_Local.java</file>
</group>
</open-files>
</project-private>
nbproject/project.properties
View file @ 03da7eb9
......@@ -3,7 +3,7 @@ annotation.processing.enabled.in.editor=false
annotation.processing.processors.list=
annotation.processing.run.all.processors=true
annotation.processing.source.output=${build.generated.sources.dir}/ap-source-output
application.title=ScigenChecker_Local
application.title=SciDetect_Local
application.vendor=tien
build.classes.dir=${build.dir}/classes
build.classes.excludes=**/*.java,**/*.form
......@@ -25,7 +25,7 @@ debug.test.classpath=\
dist.archive.excludes=
# This directory is removed when the project is cleaned:
dist.dir=dist
dist.jar=${dist.dir}/ScigenChecker_Local.jar
dist.jar=${dist.dir}/SciDetect_Local.jar
dist.javadoc.dir=${dist.dir}/javadoc
endorsed.classpath=
excludes=
......@@ -68,7 +68,7 @@ jnlp.signed=false
jnlp.signing=
jnlp.signing.alias=
jnlp.signing.keystore=
main.class=scigenchecker_local.ScigenChecker_Local
main.class=fr.imag.forge.scidetect.SciDetect_local.Scidetect_Local
# Optional override of default Codebase manifest attribute, use to prevent RIAs from being repurposed
manifest.custom.codebase=
# Optional override of default Permissions manifest attribute (supported values: sandbox, all-permissions)
......@@ -77,7 +77,7 @@ manifest.file=manifest.mf
meta.inf.dir=${src.dir}/META-INF
mkdist.disabled=false
platform.active=JDK_1.6
project.license=gpl20
project.license=gpl30
run.classpath=\
${javac.classpath}:\
${build.classes.dir}
......
nbproject/project.xml
View file @ 03da7eb9
......@@ -3,7 +3,7 @@
<type>org.netbeans.modules.java.j2seproject</type>
<configuration>
<data xmlns="http://www.netbeans.org/ns/j2se-project/3">
<name>ScigenChecker_Local</name>
<name>SciDetect_Local</name>
<explicit-platform explicit-source-supported="true"/>
<source-roots>
<root id="src.dir"/>
......
src/fr/imag/forge/scidetect/Checker/Reader.java
View file @ 03da7eb9
......@@ -20,7 +20,7 @@ package fr.imag.forge.scidetect.Checker;
//import fr.imag.forge.scidetect.TextExtractor.Xmlextractor;
//import fr.imag.forge.scidetect.TextExtractor.pdfextractor;
import fr.imag.forge.scidetect.Corpus.Corpus;
import fr.imag.forge.scidetect.Corpus.ProcessText;
import fr.imag.forge.scidetect.Corpus.TextProcessor;
import fr.imag.forge.scidetect.Corpus.Text;
import java.io.BufferedReader;
import java.io.File;
......@@ -62,7 +62,7 @@ public class Reader {
//other config should be read over here
if (b[0].equals("Max_length")) {
maxlength = Integer.parseInt(b[1]);
ProcessText.maxlength = Integer.parseInt(b[1]);
TextProcessor.maxlength = Integer.parseInt(b[1]);
}
}
}
......@@ -83,7 +83,7 @@ public class Reader {
readsamples(listOfFile[j].getPath());
} else if (listOfFile[j].getName().endsWith(".pdf") || listOfFile[j].getName().endsWith(".xml") ||listOfFile[j].getName().endsWith(".xtx")||(listOfFile[j].getName().endsWith(".txt") && !listOfFile[j].getName().startsWith("INDEX-"))) {
ArrayList<Text> text = new ArrayList<Text>();
ProcessText textprocessor = new ProcessText();
TextProcessor textprocessor = new TextProcessor();
text = textprocessor.newtext(listOfFile[j], listOfFile);
for (int i = 0; i < text.size(); i++) {
samples.put(text.get(i));
......@@ -109,7 +109,7 @@ public class Reader {
readtests(listOfFile[j].getPath());
} else if (listOfFile[j].getName().endsWith(".pdf") || listOfFile[j].getName().endsWith(".xml") ||listOfFile[j].getName().endsWith(".xtx")) {
ArrayList<Text> text = new ArrayList<Text>();
ProcessText textprocessor = new ProcessText();
TextProcessor textprocessor = new TextProcessor();
text = textprocessor.newtext(listOfFile[j], listOfFile);
for (int i = 0; i < text.size(); i++) {
test.put(text.get(i));
......
src/fr/imag/forge/scidetect/Corpus/TextProcessor.java 0 → 100644
View file @ 03da7eb9
/*
* Copyright (C) 2015 UNIVERSITE JOSEPH FOURIER (Grenoble 1)/ Springer-Verlag GmbH
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.imag.forge.scidetect.Corpus;
import fr.imag.forge.scidetect.Checker.Indexer;
import fr.imag.forge.scidetect.TextExtractor.Xmlextractor;
import fr.imag.forge.scidetect.TextExtractor.pdfextractor;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
* Manage texts file in the corpus
*
* @author Nguyen Minh Tien - minh-tien.nguyen@imag.fr
*/
public class TextProcessor {
/**
*
*/
public static int maxlength;
ArrayList<Text> text = new ArrayList<Text>();
/**
* Process a File (pdf,xml) to create clean set of text (incase of need to split)
*
* @param original file
* @param listOfFile
* @return list of text[]
* @throws java.io.IOException
*
*/
public ArrayList<Text> newtext(File original, File[] listOfFile) throws IOException {
// find if there is already index for it
String indexname = "INDEX-"
+ original.getName().substring(0,
original.getName().lastIndexOf("."))
+ ".txt";
String content = "";
if (Arrays.asList(listOfFile).toString().contains(indexname)) {
// System.out.println("lets read from index file");
readindexfile(original.getParent() + "/" + indexname);
} else {
if (original.getName().endsWith(".pdf")) {
try {
pdfextractor a = new pdfextractor();
content = a.pdfextract(original);
} catch (FileNotFoundException ex) {
Logger.getLogger(TextProcessor.class.getName()).log(Level.SEVERE, null, ex);
}
} else if (original.getName().endsWith(".xml") || original.getName().endsWith(".xtx")) {
Xmlextractor a = new Xmlextractor();
content = a.xmlextract(original);
}
//lets deal with long file over here
//split content and the index part by part
if (content.length() < maxlength) {
Indexer b = new Indexer();
b.index(content, original);
readindexfile(original.getParent() + "/" + indexname);
} else {
String[] part = splitcontent(content);
for (int i = 0; i < part.length; i++) {
String indexnameparti = "INDEX-"
+ original.getName().substring(0,
original.getName().lastIndexOf("."))
+ "_part" + i + ".txt";
String filename = original.getName().substring(0,
original.getName().lastIndexOf("."))
+ "_part" + i + ".txt";
Indexer b = new Indexer();
File a = new File(original.getParent() + "/" + filename);
PrintWriter out = new PrintWriter(new FileWriter(a));
out.println(part[i]);
//System.out.println(text);
out.close();
b.index(part[i], a);
readindexfile(a.getParent() + "/" + indexnameparti);
}
}
}
return text;
}
/**
* Read the index file if it is avaiable
* @param path to the index file
* @return hashmap of indexs
*/
private HashMap<String, Integer> readindexfile(String path) throws IOException {
File index = new File(path);
BufferedReader br;
br = new BufferedReader(new FileReader(index));
String line;
HashMap<String, Integer> a = new HashMap<String, Integer>();
while ((line = br.readLine()) != null) {
String[] b = line.split("\t");
a.put(b[0], Integer.parseInt(b[1]));
}
br.close();
Text c = new Text();
c.setindex(a);
c.setname(path);
text.add(c);
return a;
}
private String[] splitcontent(String content) {
int nbofpart = content.length() / maxlength;
String[] part = new String[nbofpart + 1];
int lower = 0;
int upper = 0;
int i;
for (i = 0; i < nbofpart; i++) {
upper += maxlength;
part[i] = content.substring(lower, upper);
lower = upper;
}
if (upper <= content.length() - 1) {
lower = upper;
upper = content.length();
part[i] = (content.substring(lower, upper));
}
return part;
}
}
src/fr/imag/forge/scidetect/SciDetect_local/SciDetect_Local.java 0 → 100644
View file @ 03da7eb9
/*
* Copyright (C) 2015 UNIVERSITE JOSEPH FOURIER (Grenoble 1)/ Springer-Verlag GmbH
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.imag.forge.scidetect.SciDetect_local;
import fr.imag.forge.scidetect.Checker.Classifier;
import fr.imag.forge.scidetect.Checker.DistantCalculator;
import fr.imag.forge.scidetect.Checker.Reader;
import fr.imag.forge.scidetect.Checker.Utils.DistancesSet;
import fr.imag.forge.scidetect.Logger.Log;
import fr.imag.forge.scidetect.Corpus.Corpus;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Date;
/**
*
* @author Nguyen Minh Tien - minh-tien.nguyen@imag.fr
*/
public class SciDetect_Local {
// private String loglocation;
// private String detailloglocation;
private String testpath;
//private String logtime;
private Corpus samples = new Corpus();
private Corpus tests = new Corpus();
private String SamplesFolder;
//private HashMap<String, HashMap<String, Double>> distant = new HashMap<String, HashMap<String, Double>>();
DistancesSet distant = new DistancesSet();
private Boolean savedetaillog = false;
/**
* Read in the config file:
*- places where to find samples of each class
*- default places where to write results.
* @throws FileNotFoundException
* @throws IOException
*/
private void readconfig() throws FileNotFoundException, IOException {
File conf = new File("config.txt");
BufferedReader br = new BufferedReader(new FileReader(conf));
String line;
while ((line = br.readLine()) != null) {
if (!line.startsWith("#")) {
// System.out.println(line);
String[] b = line.split("\t");
if (b[0].equals("samples")) {
SamplesFolder = b[1];
}
if (b[0].equals("Default_log_folder")) {
Log.loglocation = b[1];
// System.out.println(loglocation);
}
if (b[0].equals("Default_detail_log_folder")) {
Log.detailloglocation = b[1];
//System.out.println(detailloglocation);
}
}
}
}
/**
* @throws IOException
*/
private void compute() throws IOException {
if (testpath != null) {
DateFormat dateFormat = new SimpleDateFormat("HH:mm dd.MM.yyyy");
Date date = new Date();
Log.logtime = dateFormat.format(date);
try {
Reader reader = new Reader();
reader.readconfig();
samples = reader.readsamples(SamplesFolder);
tests = reader.readtests(testpath);
} catch (Exception e) {
System.out.println("* Something went wrong during:");
System.out.println(" - reading the config file");
System.out.println(" - or reading the samples (dir data)");
System.out.println(" - or txt extraction from pdf");
System.out.println("* Continuing anyway...");
//e.printStackTrace();
}
DistantCalculator dc = new DistantCalculator();
distant = dc.caldistant(samples, tests);
Classifier cl = new Classifier();
String conclusion = cl.classify(distant);
System.out.println(conclusion);
Log log = new Log();
log.savelog(conclusion);
if (savedetaillog) {
log.savedetaillog(distant);
}
} else {
System.out.println("***** Can not read path to the folder:"+testpath);
System.out.println("***** The folder should contains file to check");
}
}
/**
* Parsing of the command line arguments:
* where to find pdf files, where results should be written
* @param args
*/
public void readargs(String[] args) {
if (args.length > 0) {
for (int i = 0; i < args.length; i += 1) {
// System.out.println(args[i]);
if (args[i].equals("-l")) {
Log.loglocation = args[i + 1];
}
if (args[i].equals("-c")) {
testpath = args[i + 1];
}
if (args[i].equals("-d")) {
savedetaillog = true;
}
if (args[i].equals("-h")) {
printUsage();
}
}
} else
{printUsage();}
}
/**
* To print usage (-h)
*/
private static void printUsage() {
System.out.println("***** Scigen & Co Checker \n");
System.out.println("To test all files in a directory <pathToFilesDirToTest>:");
System.out.println("java -jar ScigenChecker_local.jar -l <pathToLogFile> -c <pathToFilesDirToTest> \n");
System.out.println("To print usage:");
System.out.println("java -jar ScigenChecker_local.jar -h \n");
System.out.println("***** \n");
}
/**
* This is the standalone checker. All pdf files in the dir specified after -c are
* checked against classes found in the dir "data". Results are written in the log
* file specified by the -l option. If -d is given a detailled log is produced.
* Example: testing all pdf files in a directory MyConf/PDF/ and having results
* in the MyConf/checklog.txt:
* java -jar ScigenChecker_local.jar -l MyConf/checklog.txt -c MyConf/PDF/
* @param args the command line arguments
* @throws java.io.IOException
*/
public static void main(String[] args) throws IOException {
SciDetect_Local a = new SciDetect_Local();
a.readconfig();
a.readargs(args);
a.compute();
}
}
src/fr/imag/forge/scidetect/SciDetect_local/Scidetect_Local.java 0 → 100644
View file @ 03da7eb9
/*
* Copyright (C) 2015 UNIVERSITE JOSEPH FOURIER (Grenoble 1)/ Springer-Verlag GmbH
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.imag.forge.scidetect.SciDetect_local;
import fr.imag.forge.scidetect.Checker.Classifier;
import fr.imag.forge.scidetect.Checker.DistantCalculator;
import fr.imag.forge.scidetect.Checker.Reader;
import fr.imag.forge.scidetect.Checker.Utils.DistancesSet;
import fr.imag.forge.scidetect.Logger.Log;
import fr.imag.forge.scidetect.Corpus.Corpus;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Date;
/**
*
* @author Nguyen Minh Tien - minh-tien.nguyen@imag.fr
*/
public class Scidetect_Local {