Skip to content

GitLab

S
scidetect
Commit 7d4318f2 authored by Tien's avatar Tien
Browse files
Options

fixed out of memory when read abnormal big folder

parent d000b9e9
Hide whitespace changes
Inline Side-by-side
Showing with 642 additions and 27 deletions
src/fr/imag/forge/scidetect/Checker/Reader.java
View file @ 7d4318f2
......@@ -19,9 +19,11 @@ package fr.imag.forge.scidetect.Checker;
//import com.sun.corba.se.spi.transport.CorbaAcceptor;
//import fr.imag.forge.scidetect.TextExtractor.Xmlextractor;
//import fr.imag.forge.scidetect.TextExtractor.pdfextractor;
import fr.imag.forge.scidetect.Checker.Utils.DistancesSet;
import fr.imag.forge.scidetect.Corpus.Corpus;
import fr.imag.forge.scidetect.Corpus.TextProcessor;
import fr.imag.forge.scidetect.Corpus.Text;
import fr.imag.forge.scidetect.Logger.Log;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
......@@ -107,14 +109,14 @@ public class Reader {
* @return test corpus
* @throws IOException
*/
public Corpus readtests(String foldername) throws IOException {
public Corpus readtests(String foldername, Corpus Samplecorpus, Boolean savedetaillog) throws IOException {
File folder = new File(foldername);
if (folder.isDirectory()) {
File[] listOfFile = folder.listFiles();
for (int j = 0; j < listOfFile.length; j++) {
if (listOfFile[j].isDirectory()) {
readtests(listOfFile[j].getPath());
readtests(listOfFile[j].getPath(), Samplecorpus, savedetaillog);
} else if (listOfFile[j].getName().endsWith(".pdf") || listOfFile[j].getName().endsWith(".xml") || listOfFile[j].getName().endsWith(".xtx")) {
ArrayList<Text> text = new ArrayList<Text>();
TextProcessor textprocessor = new TextProcessor();
......@@ -122,6 +124,18 @@ public class Reader {
for (int i = 0; i < text.size(); i++) {
test.put(text.get(i));
}
DistancesSet distant = new DistancesSet();
DistantCalculator dc = new DistantCalculator();
distant = dc.caldistant(Samplecorpus, test);
Classifier cl = new Classifier();
String conclusion = cl.classify(distant);
System.out.println(conclusion);
Log log = new Log();
log.savelog(conclusion);
if (savedetaillog) {
log.savedetaillog(distant);
}
}
}
......@@ -130,11 +144,23 @@ public class Reader {
TextProcessor textprocessor = new TextProcessor();
File[] listOfFile = folder.getParentFile().listFiles();
//listOfFile[0] = folder;
text = textprocessor.newtext(folder, listOfFile);
for (int i = 0; i < text.size(); i++) {
test.put(text.get(i));
}
DistancesSet distant = new DistancesSet();
DistantCalculator dc = new DistantCalculator();
distant = dc.caldistant(Samplecorpus, test);
Classifier cl = new Classifier();
String conclusion = cl.classify(distant);
System.out.println(conclusion);
Log log = new Log();
log.savelog(conclusion);
if (savedetaillog) {
log.savedetaillog(distant);
}
}
return test;
......
src/fr/imag/forge/scidetect/SciDetect_local/SciDetect_Local.java
View file @ 7d4318f2
......@@ -39,11 +39,11 @@ import java.util.Date;
*/
public class SciDetect_Local {
// private String loglocation;
// private String loglocation;
// private String detailloglocation;
private String testpath;
//private String logtime;
private Corpus samples = new Corpus();
public Corpus samples = new Corpus();
private Corpus tests = new Corpus();
private String SamplesFolder;
//private HashMap<String, HashMap<String, Double>> distant = new HashMap<String, HashMap<String, Double>>();
......@@ -89,9 +89,10 @@ public class SciDetect_Local {
*
* @throws IOException
*/
public void compute(String[] args) throws IOException {
public String compute(String[] args) throws IOException {
readconfig();
readargs(args);
String conclusion = new String();
if (testpath != null) {
DateFormat dateFormat = new SimpleDateFormat("yyyy.MM.dd-HH:mm");
Date date = new Date();
......@@ -101,7 +102,7 @@ public class SciDetect_Local {
reader.readconfig();
samples = reader.readsamples(SamplesFolder);
tests = reader.readtests(testpath);
tests = reader.readtests(testpath, samples, savedetaillog);
} catch (Exception e) {
System.out.println("* Something went wrong during:");
......@@ -111,24 +112,28 @@ public class SciDetect_Local {
System.out.println("* Continuing anyway...");
//e.printStackTrace();
}
DistantCalculator dc = new DistantCalculator();
distant = dc.caldistant(samples, tests);
Classifier cl = new Classifier();
String conclusion = cl.classify(distant);
System.out.println(conclusion);
Log log = new Log();
log.savelog(conclusion);
if (clean){
Cleaner cleaner = new Cleaner();
cleaner.clean(testpath);}
if (savedetaillog) {
log.savedetaillog(distant);
if (clean) {
Cleaner cleaner = new Cleaner();
cleaner.clean(testpath);
}
// DistantCalculator dc = new DistantCalculator();
// distant = dc.caldistant(samples, tests);
// Classifier cl = new Classifier();
// conclusion = cl.classify(distant);
// System.out.println(conclusion);
// Log log = new Log();
// log.savelog(conclusion);
// if (clean){
// Cleaner cleaner = new Cleaner();
// cleaner.clean(testpath);}
// if (savedetaillog) {
// log.savedetaillog(distant);
// }
} else {
System.out.println("***** Can not read path to the folder:" + testpath);
System.out.println("***** The folder should contains file to check");
}
return conclusion;
}
/**
......@@ -150,7 +155,7 @@ public class SciDetect_Local {
if (args[i].equals("-d")) {
savedetaillog = true;
}
if (args[i].equals("-noclean")) {
if (args[i].equals("-noclean")) {
clean = false;
}
if (args[i].equals("-h")) {
......
src/fr/imag/forge/scidetect/TextExtractor/normalizer.java
View file @ 7d4318f2
......@@ -51,12 +51,14 @@ public class normalizer {
}
br.close();
content = content.toUpperCase();
content = content.replaceAll("-", " ");// parenthesis
content = content.replaceAll("[^A-Z ]", "");// non A to Z
content = content.replaceAll("\n", " ");//prob not nessesary :D
content = content.replaceAll("\\s+", " ");// remove extra spaces
// content = content.replaceAll("-", " ");// parenthesis
// content = content.replaceAll("[^A-Z ]", "");// non A to Z
//
// content = content.replaceAll("\n", " ");//prob not nessesary :D
// content = content.replaceAll("\\s+", " ");// remove extra spaces\
content = content.replaceAll("[-\r\n\\s+]", " ");// parenthesis
//content = content.replaceAll("\r", " "); // make a new line
content = content.replaceAll("[^A-Z ]", "");// remove non A to Z
PrintWriter out = new PrintWriter(txt);
out.println(content);
out.close();
......
src/fr/imag/forge/scidetect/gui/FileChoser.form 0 → 100644
View file @ 7d4318f2
<?xml version="1.0" encoding="UTF-8" ?>
<Form version="1.3" maxVersion="1.9" type="org.netbeans.modules.form.forminfo.JFrameFormInfo">
<Properties>
<Property name="defaultCloseOperation" type="int" value="3"/>
</Properties>
<SyntheticProperties>
<SyntheticProperty name="formSizePolicy" type="int" value="1"/>
<SyntheticProperty name="generateCenter" type="boolean" value="false"/>
</SyntheticProperties>
<AuxValues>
<AuxValue name="FormSettings_autoResourcing" type="java.lang.Integer" value="0"/>
<AuxValue name="FormSettings_autoSetComponentName" type="java.lang.Boolean" value="false"/>
<AuxValue name="FormSettings_generateFQN" type="java.lang.Boolean" value="true"/>
<AuxValue name="FormSettings_generateMnemonicsCode" type="java.lang.Boolean" value="false"/>
<AuxValue name="FormSettings_i18nAutoMode" type="java.lang.Boolean" value="false"/>
<AuxValue name="FormSettings_layoutCodeTarget" type="java.lang.Integer" value="1"/>
<AuxValue name="FormSettings_listenerGenerationStyle" type="java.lang.Integer" value="0"/>
<AuxValue name="FormSettings_variablesLocal" type="java.lang.Boolean" value="false"/>
<AuxValue name="FormSettings_variablesModifier" type="java.lang.Integer" value="2"/>
</AuxValues>
<Layout>
<DimensionLayout dim="0">
<Group type="103" groupAlignment="0" attributes="0">
<EmptySpace min="0" pref="400" max="32767" attributes="0"/>
</Group>
</DimensionLayout>
<DimensionLayout dim="1">
<Group type="103" groupAlignment="0" attributes="0">
<EmptySpace min="0" pref="300" max="32767" attributes="0"/>
</Group>
</DimensionLayout>
</Layout>
</Form>
src/fr/imag/forge/scidetect/gui/FileChoser.java 0 → 100644
View file @ 7d4318f2
/*
* Copyright (C) 2015 tien
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.imag.forge.scidetect.gui;
/**
*
* @author tien
*/
public class FileChoser extends javax.swing.JFrame {
/**
* Creates new form FileChoser
*/
public FileChoser() {
initComponents();
}
/**
* This method is called from within the constructor to initialize the form.
* WARNING: Do NOT modify this code. The content of this method is always
* regenerated by the Form Editor.
*/
@SuppressWarnings("unchecked")
// <editor-fold defaultstate="collapsed" desc="Generated Code">//GEN-BEGIN:initComponents
private void initComponents() {
setDefaultCloseOperation(javax.swing.WindowConstants.EXIT_ON_CLOSE);
javax.swing.GroupLayout layout = new javax.swing.GroupLayout(getContentPane());
getContentPane().setLayout(layout);
layout.setHorizontalGroup(
layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addGap(0, 400, Short.MAX_VALUE)
);
layout.setVerticalGroup(
layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addGap(0, 300, Short.MAX_VALUE)
);
pack();
}// </editor-fold>//GEN-END:initComponents
/**
* @param args the command line arguments
*/
public static void main(String args[]) {
/* Set the Nimbus look and feel */
//<editor-fold defaultstate="collapsed" desc=" Look and feel setting code (optional) ">
/* If Nimbus (introduced in Java SE 6) is not available, stay with the default look and feel.
* For details see http://download.oracle.com/javase/tutorial/uiswing/lookandfeel/plaf.html
*/
try {
for (javax.swing.UIManager.LookAndFeelInfo info : javax.swing.UIManager.getInstalledLookAndFeels()) {
if ("Nimbus".equals(info.getName())) {
javax.swing.UIManager.setLookAndFeel(info.getClassName());
break;
}
}
} catch (ClassNotFoundException ex) {
java.util.logging.Logger.getLogger(FileChoser.class.getName()).log(java.util.logging.Level.SEVERE, null, ex);
} catch (InstantiationException ex) {
java.util.logging.Logger.getLogger(FileChoser.class.getName()).log(java.util.logging.Level.SEVERE, null, ex);
} catch (IllegalAccessException ex) {
java.util.logging.Logger.getLogger(FileChoser.class.getName()).log(java.util.logging.Level.SEVERE, null, ex);
} catch (javax.swing.UnsupportedLookAndFeelException ex) {
java.util.logging.Logger.getLogger(FileChoser.class.getName()).log(java.util.logging.Level.SEVERE, null, ex);
}
//</editor-fold>
/* Create and display the form */
java.awt.EventQueue.invokeLater(new Runnable() {
public void run() {
new FileChoser().setVisible(true);
}
});
}
// Variables declaration - do not modify//GEN-BEGIN:variables
// End of variables declaration//GEN-END:variables
}
src/fr/imag/forge/scidetect/gui/SciDetect_GUI.form 0 → 100644
View file @ 7d4318f2
<?xml version="1.0" encoding="UTF-8" ?>
<Form version="1.5" maxVersion="1.9" type="org.netbeans.modules.form.forminfo.JFrameFormInfo">
<NonVisualComponents>
<Component class="javax.swing.JFileChooser" name="jFileChooser1">
<Properties>
<Property name="fileFilter" type="javax.swing.filechooser.FileFilter" editor="org.netbeans.modules.form.RADConnectionPropertyEditor">
<Connection code="new MyCustomFilter()" type="code"/>
</Property>
</Properties>
</Component>
</NonVisualComponents>
<Properties>
<Property name="defaultCloseOperation" type="int" value="3"/>
</Properties>
<SyntheticProperties>
<SyntheticProperty name="formSizePolicy" type="int" value="1"/>
<SyntheticProperty name="generateCenter" type="boolean" value="false"/>
</SyntheticProperties>
<AuxValues>
<AuxValue name="FormSettings_autoResourcing" type="java.lang.Integer" value="0"/>
<AuxValue name="FormSettings_autoSetComponentName" type="java.lang.Boolean" value="false"/>
<AuxValue name="FormSettings_generateFQN" type="java.lang.Boolean" value="true"/>
<AuxValue name="FormSettings_generateMnemonicsCode" type="java.lang.Boolean" value="false"/>
<AuxValue name="FormSettings_i18nAutoMode" type="java.lang.Boolean" value="false"/>
<AuxValue name="FormSettings_layoutCodeTarget" type="java.lang.Integer" value="1"/>
<AuxValue name="FormSettings_listenerGenerationStyle" type="java.lang.Integer" value="0"/>
<AuxValue name="FormSettings_variablesLocal" type="java.lang.Boolean" value="false"/>
<AuxValue name="FormSettings_variablesModifier" type="java.lang.Integer" value="2"/>
</AuxValues>
<Layout>
<DimensionLayout dim="0">
<Group type="103" groupAlignment="0" attributes="0">
<Group type="102" alignment="0" attributes="0">
<EmptySpace max="-2" attributes="0"/>
<Component id="jScrollPane1" min="-2" pref="587" max="-2" attributes="0"/>
<EmptySpace type="separate" max="-2" attributes="0"/>
<Group type="103" groupAlignment="0" attributes="0">
<Group type="102" attributes="0">
<Group type="103" groupAlignment="1" attributes="0">
<Component id="jButton3" max="32767" attributes="0"/>
<Component id="jButton1" max="32767" attributes="0"/>
<Component id="jButton2" max="32767" attributes="0"/>
</Group>
<EmptySpace min="-2" pref="25" max="-2" attributes="0"/>
</Group>
<Group type="102" attributes="0">
<Group type="103" groupAlignment="0" attributes="0">
<Component id="jCheckBox1" min="-2" max="-2" attributes="0"/>
<Component id="jCheckBox2" min="-2" max="-2" attributes="0"/>
</Group>
<EmptySpace max="32767" attributes="0"/>
</Group>
</Group>
</Group>
</Group>
</DimensionLayout>
<DimensionLayout dim="1">
<Group type="103" groupAlignment="0" attributes="0">
<Group type="102" alignment="0" attributes="0">
<EmptySpace max="-2" attributes="0"/>
<Group type="103" groupAlignment="0" attributes="0">
<Group type="102" attributes="0">
<Component id="jScrollPane1" min="-2" max="-2" attributes="0"/>
<EmptySpace min="0" pref="5" max="32767" attributes="0"/>
</Group>
<Group type="102" attributes="0">
<Component id="jButton1" min="-2" max="-2" attributes="0"/>
<EmptySpace min="-2" pref="29" max="-2" attributes="0"/>
<Component id="jCheckBox1" min="-2" max="-2" attributes="0"/>
<EmptySpace min="-2" pref="23" max="-2" attributes="0"/>
<Component id="jCheckBox2" min="-2" max="-2" attributes="0"/>
<EmptySpace type="separate" max="-2" attributes="0"/>
<Component id="jButton2" min="-2" max="-2" attributes="0"/>
<EmptySpace max="32767" attributes="0"/>
<Component id="jButton3" min="-2" max="-2" attributes="0"/>
</Group>
</Group>
<EmptySpace max="-2" attributes="0"/>
</Group>
</Group>
</DimensionLayout>
</Layout>
<SubComponents>
<Container class="javax.swing.JScrollPane" name="jScrollPane1">
<AuxValues>
<AuxValue name="autoScrollPane" type="java.lang.Boolean" value="true"/>
</AuxValues>
<Layout class="org.netbeans.modules.form.compat2.layouts.support.JScrollPaneSupportLayout"/>
<SubComponents>
<Component class="javax.swing.JTable" name="jTable1">
<Properties>
<Property name="model" type="javax.swing.table.TableModel" editor="org.netbeans.modules.form.editors2.TableModelEditor">
<Table columnCount="3" rowCount="0">
<Column editable="false" title="File Name" type="java.lang.Object"/>
<Column editable="false" title="Result" type="java.lang.Object"/>
<Column editable="false" title="Distance" type="java.lang.Object"/>
</Table>
</Property>
<Property name="columnModel" type="javax.swing.table.TableColumnModel" editor="org.netbeans.modules.form.editors2.TableColumnModelEditor">
<TableColumnModel selectionModel="0">
<Column maxWidth="-1" minWidth="-1" prefWidth="-1" resizable="true">
<Title/>
<Editor/>
<Renderer/>
</Column>
<Column maxWidth="-1" minWidth="-1" prefWidth="-1" resizable="true">
<Title/>
<Editor/>
<Renderer/>
</Column>
<Column maxWidth="-1" minWidth="-1" prefWidth="-1" resizable="true">
<Title/>
<Editor/>
<Renderer/>
</Column>
</TableColumnModel>
</Property>
<Property name="tableHeader" type="javax.swing.table.JTableHeader" editor="org.netbeans.modules.form.editors2.JTableHeaderEditor">
<TableHeader reorderingAllowed="true" resizingAllowed="true"/>
</Property>
</Properties>
</Component>
</SubComponents>
</Container>
<Component class="javax.swing.JButton" name="jButton1">
<Properties>
<Property name="text" type="java.lang.String" value="Add"/>
</Properties>
<Events>
<EventHandler event="actionPerformed" listener="java.awt.event.ActionListener" parameters="java.awt.event.ActionEvent" handler="jButton1ActionPerformed"/>
</Events>
</Component>
<Component class="javax.swing.JButton" name="jButton2">
<Properties>
<Property name="text" type="java.lang.String" value="Scan"/>
</Properties>
<Events>
<EventHandler event="actionPerformed" listener="java.awt.event.ActionListener" parameters="java.awt.event.ActionEvent" handler="jButton2ActionPerformed"/>
</Events>
</Component>
<Component class="javax.swing.JButton" name="jButton3">
<Properties>
<Property name="text" type="java.lang.String" value="Close"/>
<Property name="toolTipText" type="java.lang.String" value=""/>
</Properties>
<Events>
<EventHandler event="actionPerformed" listener="java.awt.event.ActionListener" parameters="java.awt.event.ActionEvent" handler="jButton3ActionPerformed"/>
</Events>
</Component>
<Component class="javax.swing.JCheckBox" name="jCheckBox1">
<Properties>
<Property name="text" type="java.lang.String" value="Detail Log"/>
</Properties>
</Component>
<Component class="javax.swing.JCheckBox" name="jCheckBox2">
<Properties>
<Property name="selected" type="boolean" value="true"/>
<Property name="text" type="java.lang.String" value="Clean up"/>
</Properties>
</Component>
</SubComponents>
</Form>
src/fr/imag/forge/scidetect/gui/SciDetect_GUI.java 0 → 100644
View file @ 7d4318f2
/*
* Copyright (C) 2015 tien
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.imag.forge.scidetect.gui;
import fr.imag.forge.scidetect.SciDetect_local.SciDetect_Local;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import javax.swing.JFileChooser;
import javax.swing.JOptionPane;
import javax.swing.table.DefaultTableModel;
/**
*
* @author tien
*/
public class SciDetect_GUI extends javax.swing.JFrame {
/**
* Creates new form SciDetect_GUI
*/
public SciDetect_GUI() {
initComponents();
}
/**
* This method is called from within the constructor to initialize the form.
* WARNING: Do NOT modify this code. The content of this method is always
* regenerated by the Form Editor.
*/
@SuppressWarnings("unchecked")
// <editor-fold defaultstate="collapsed" desc="Generated Code">//GEN-BEGIN:initComponents
private void initComponents() {
jFileChooser1 = new javax.swing.JFileChooser();
jScrollPane1 = new javax.swing.JScrollPane();
jTable1 = new javax.swing.JTable();
jButton1 = new javax.swing.JButton();
jButton2 = new javax.swing.JButton();
jButton3 = new javax.swing.JButton();
jCheckBox1 = new javax.swing.JCheckBox();
jCheckBox2 = new javax.swing.JCheckBox();
jFileChooser1.setFileFilter(new MyCustomFilter());
setDefaultCloseOperation(javax.swing.WindowConstants.EXIT_ON_CLOSE);
jTable1.setModel(new javax.swing.table.DefaultTableModel(
new Object [][] {
},
new String [] {
"File Name", "Result", "Distance"
}
) {
boolean[] canEdit = new boolean [] {
false, false, false
};
public boolean isCellEditable(int rowIndex, int columnIndex) {
return canEdit [columnIndex];
}
});
jScrollPane1.setViewportView(jTable1);
jButton1.setText("Add");
jButton1.addActionListener(new java.awt.event.ActionListener() {
public void actionPerformed(java.awt.event.ActionEvent evt) {
jButton1ActionPerformed(evt);
}
});
jButton2.setText("Scan");
jButton2.addActionListener(new java.awt.event.ActionListener() {
public void actionPerformed(java.awt.event.ActionEvent evt) {
jButton2ActionPerformed(evt);
}
});
jButton3.setText("Close");
jButton3.setToolTipText("");
jButton3.addActionListener(new java.awt.event.ActionListener() {
public void actionPerformed(java.awt.event.ActionEvent evt) {
jButton3ActionPerformed(evt);
}
});
jCheckBox1.setText("Detail Log");
jCheckBox2.setSelected(true);
jCheckBox2.setText("Clean up");
javax.swing.GroupLayout layout = new javax.swing.GroupLayout(getContentPane());
getContentPane().setLayout(layout);
layout.setHorizontalGroup(
layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addGroup(layout.createSequentialGroup()
.addContainerGap()
.addComponent(jScrollPane1, javax.swing.GroupLayout.PREFERRED_SIZE, 587, javax.swing.GroupLayout.PREFERRED_SIZE)
.addGap(18, 18, 18)