Commit 3be8aad5 authored by Sylvain Hatier's avatar Sylvain Hatier Committed by Arnaud Bey
Browse files

WIP : intégration new lexicon & old MW

parent 9b807649
/logs
/.data
.env
.db
application/.web-server-pid
application/app/config/parameters.yml
......@@ -33,4 +33,5 @@ application/web/upload/profilepic/*
!application/web/upload/profilepic/.gitkeep
application/web/files/*
!aplication/web/files/.gitkeep
*.php~
application/data/
*.php~
......@@ -22,8 +22,8 @@ vi .env
#vi or any other text editor, this will allow you to set the DB root password
#this password will be asked during the install
sudo docker-compose up --build -d
sudo docker-compose exec php bash
PWD=dbrootpassword
sudo docker-compose exec apache bash
PWD=root
#(optional) this command is to allow you not to enter the db root password (set in .env) for each query that it is supposed to perform
make install
```
......
magicword
=========
# Notes
- Mariadb doit prendre en compte le fichier de conf dans docker-files
- vérifier dans interface adminer, dans /variables, valeur token et stopword
- Pour import et suppression de lexiques -> le faire en commande car long
install: composer npm permission dbcreate dbpopulate dbupdate assets cache
install: composer npm permission dbupdate assets cache
update: composer npm permission dbupdate assets cache
dbcreate:
php bin/console doctrine:database:create --if-not-exists
php bin/console doctrine:database:create --connection=lexicon --if-not-exists
dbpopulate:
mysql -u root -p$(PWD) -h db mw < mw_fixtures.sql
mysql -u root -p$(PWD) -h db lexicon < lexicon.sql
mysql -u root -proot -h db dbmw < mw_fixtures.sql
dbupdate:
php bin/console doctrine:schema:update --force
......
......@@ -69,16 +69,6 @@ doctrine:
charset: UTF8
logging: '%kernel.debug%'
profiling: '%kernel.debug%'
lexicon:
driver: pdo_mysql
host: "%database_host%"
port: "%database_port%"
dbname: "%database_name2%"
user: "%database_user%"
password: "%database_password%"
charset: UTF8
logging: '%kernel.debug%'
profiling: '%kernel.debug%'
orm:
auto_generate_proxy_classes: "%kernel.debug%"
......@@ -93,13 +83,6 @@ doctrine:
mappings:
MagicWordBundle: ~
LexiconBundle: ~
lexicon:
connection: lexicon
mappings:
LexiconBundle:
mapping: true
is_bundle: true
type: annotation
# Swiftmailer Configuration
swiftmailer:
......
......@@ -7,7 +7,6 @@ parameters:
database_name: mw
database_user: root
database_password: root
database_name2: lexicon
mailer_transport: smtp
mailer_host: 127.0.0.1
mailer_user: default_user
......
This diff is collapsed.
......@@ -108,120 +108,6 @@ INSERT INTO `language_ui` (`id`, `name`, `value`) VALUES
(1, 'french', 'fr'),
(2, 'english', 'en');
-- --------------------------------------------------------
--
-- Structure de la table `letter_canonic_letter`
--
CREATE TABLE IF NOT EXISTS `letter_canonic_letter` (
`id` int(11) NOT NULL,
`value` varchar(1) COLLATE utf8_unicode_ci NOT NULL
) ENGINE=InnoDB AUTO_INCREMENT=27 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
--
-- Contenu de la table `letter_canonic_letter`
--
INSERT INTO `letter_canonic_letter` (`id`, `value`) VALUES
(1, 'a'),
(2, 'b'),
(3, 'c'),
(4, 'd'),
(5, 'e'),
(6, 'f'),
(7, 'g'),
(8, 'h'),
(9, 'i'),
(10, 'j'),
(11, 'k'),
(12, 'l'),
(13, 'm'),
(14, 'n'),
(15, 'o'),
(16, 'p'),
(17, 'q'),
(18, 'r'),
(19, 's'),
(20, 't'),
(21, 'u'),
(22, 'v'),
(23, 'w'),
(24, 'x'),
(25, 'y'),
(26, 'z');
-- --------------------------------------------------------
--
-- Structure de la table `letter_language`
--
CREATE TABLE IF NOT EXISTS `letter_language` (
`id` int(11) NOT NULL,
`letter_id` int(11) DEFAULT NULL,
`point` int(11) NOT NULL,
`weight` int(11) NOT NULL,
`language_id` int(11) DEFAULT NULL
) ENGINE=InnoDB AUTO_INCREMENT=64 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
--
-- Contenu de la table `letter_language`
--
INSERT INTO `letter_language` (`id`, `letter_id`, `point`, `weight`, `language_id`) VALUES
(1, 1, 1, 9, 1),
(2, 2, 3, 2, 1),
(14, 3, 3, 2, 1),
(15, 4, 2, 3, 1),
(16, 5, 1, 15, 1),
(17, 6, 4, 2, 1),
(18, 7, 2, 2, 1),
(19, 8, 4, 2, 1),
(20, 9, 1, 8, 1),
(21, 10, 8, 1, 1),
(22, 11, 10, 1, 1),
(23, 12, 1, 5, 1),
(24, 13, 2, 3, 1),
(25, 14, 1, 6, 1),
(26, 15, 1, 6, 1),
(27, 16, 3, 2, 1),
(28, 17, 8, 1, 1),
(29, 18, 1, 6, 1),
(30, 19, 1, 6, 1),
(31, 20, 1, 6, 1),
(32, 21, 1, 6, 1),
(33, 22, 4, 2, 1),
(34, 23, 10, 1, 1),
(35, 24, 10, 1, 1),
(36, 25, 10, 1, 1),
(37, 26, 10, 1, 1),
(38, 1, 1, 9, 2),
(39, 2, 3, 2, 2),
(40, 3, 3, 2, 2),
(41, 4, 2, 4, 2),
(42, 5, 1, 12, 2),
(43, 6, 4, 2, 2),
(44, 7, 2, 3, 2),
(45, 8, 4, 2, 2),
(46, 9, 1, 9, 2),
(47, 10, 8, 1, 2),
(48, 11, 5, 1, 2),
(49, 12, 1, 4, 2),
(50, 13, 3, 2, 2),
(51, 14, 1, 6, 2),
(52, 15, 1, 8, 2),
(53, 16, 3, 2, 2),
(54, 17, 10, 1, 2),
(55, 18, 1, 6, 2),
(56, 19, 1, 4, 2),
(57, 20, 1, 6, 2),
(58, 21, 1, 4, 2),
(59, 22, 4, 2, 2),
(60, 23, 4, 2, 2),
(61, 24, 8, 1, 2),
(62, 25, 4, 2, 2),
(63, 26, 1, 10, 2);
-- --------------------------------------------------------
......@@ -261,38 +147,7 @@ CREATE TABLE IF NOT EXISTS `wordbox_acquisition_type` (
INSERT INTO `wordbox_acquisition_type` (`id`, `value`) VALUES
(1, 'manual');
-- --------------------------------------------------------
--
-- Structure de la table `word_length_points`
--
CREATE TABLE IF NOT EXISTS `word_length_points` (
`id` int(11) NOT NULL,
`length` int(11) NOT NULL,
`points` int(11) NOT NULL
) ENGINE=InnoDB AUTO_INCREMENT=16 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
--
-- Contenu de la table `word_length_points`
--
INSERT INTO `word_length_points` (`id`, `length`, `points`) VALUES
(1, 2, 0),
(2, 3, 1),
(3, 4, 1),
(4, 5, 2),
(5, 6, 3),
(6, 7, 5),
(7, 8, 11),
(8, 9, 12),
(9, 10, 13),
(10, 11, 14),
(11, 12, 15),
(12, 13, 20),
(13, 14, 20),
(14, 15, 40),
(15, 16, 50);
--
-- Index pour les tables exportées
......@@ -322,18 +177,6 @@ ALTER TABLE `general_parameters`
ALTER TABLE `language_ui`
ADD PRIMARY KEY (`id`);
--
-- Index pour la table `letter_canonic_letter`
--
ALTER TABLE `letter_canonic_letter`
ADD PRIMARY KEY (`id`), ADD UNIQUE KEY `UNIQ_ED1363F21D775834` (`value`);
--
-- Index pour la table `letter_language`
--
ALTER TABLE `letter_language`
ADD PRIMARY KEY (`id`), ADD KEY `IDX_4A4AED994525FF26` (`letter_id`), ADD KEY `IDX_4A4AED9982F1BAF4` (`language_id`);
--
-- Index pour la table `roundType`
--
......@@ -346,12 +189,6 @@ ALTER TABLE `roundType`
ALTER TABLE `wordbox_acquisition_type`
ADD PRIMARY KEY (`id`);
--
-- Index pour la table `word_length_points`
--
ALTER TABLE `word_length_points`
ADD PRIMARY KEY (`id`);
--
-- AUTO_INCREMENT pour les tables exportées
--
......@@ -376,16 +213,7 @@ MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=2;
--
ALTER TABLE `language_ui`
MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=3;
--
-- AUTO_INCREMENT pour la table `letter_canonic_letter`
--
ALTER TABLE `letter_canonic_letter`
MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=27;
--
-- AUTO_INCREMENT pour la table `letter_language`
--
ALTER TABLE `letter_language`
MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=64;
--
-- AUTO_INCREMENT pour la table `roundType`
--
......@@ -396,14 +224,6 @@ MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=4;
--
ALTER TABLE `wordbox_acquisition_type`
MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=2;
--
-- AUTO_INCREMENT pour la table `word_length_points`
--
ALTER TABLE `word_length_points`
MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=16;
--
-- Contraintes pour les tables exportées
--
--
-- Contraintes pour la table `general_parameters`
......@@ -411,13 +231,6 @@ MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=16;
ALTER TABLE `general_parameters`
ADD CONSTRAINT `FK_75AC6A6F89366B7B` FOREIGN KEY (`tutorial_id`) REFERENCES `game` (`id`);
--
-- Contraintes pour la table `letter_language`
--
ALTER TABLE `letter_language`
ADD CONSTRAINT `FK_4A4AED994525FF26` FOREIGN KEY (`letter_id`) REFERENCES `letter_canonic_letter` (`id`);
SET FOREIGN_KEY_CHECKS=1;
/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
<?php
namespace LexiconBundle\Command;
use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
use MagicWordBundle\Entity\Lexicon\Lemma;
use MagicWordBundle\Entity\Lexicon\Inflection;
class EnglishImportCommand extends ContainerAwareCommand
{
protected function configure()
{
$this
->setName('lexicon:import-english')
->setDescription('import english')
;
}
protected function execute(InputInterface $input, OutputInterface $output)
{
$em = $this->getContainer()->get('doctrine')->getEntityManager('default');
$enForms = $em->getRepository('LexiconBundle:Inflection')->getEnglishForms();
foreach ($enForms as $enForm) {
$lemma = $this->handleLemma($enForm, $output);
if ($lemma) {
preg_match_all('/:[^:]+/', $enForm['features'], $features);
foreach ($features as $feature) {
foreach ($feature as $f) {
$inflection = $this->handleInflection($f, $lemma, $enForm['form'], $output);
}
}
$em->clear();
}
}
}
private function handleInflection($subcat, $lemma, $form, $output)
{
$em = $this->getContainer()->get('doctrine')->getEntityManager('default');
$inflectionManager = $this->getContainer()->get('lexicon_inflection');
$genderNames = array(
'm' => 'masculine',
'f' => 'feminine',
'n' => 'neutral',
);
$numberNames = array(
's' => 'singular',
'p' => 'plural',
);
$personNames = array(
'1' => 'firstPerson',
'2' => 'secondPerson',
'3' => 'thirdPerson',
);
$tenseNames = array(
'W' => null,
'P' => 'present',
'K' => 'past',
'I' => 'simplePast',
'G' => 'present',
'C' => null,
'S' => null,
);
$moodNames = array(
'W' => 'infinitive',
'P' => 'indicative',
'K' => 'participle',
'I' => 'indicative',
'G' => 'participle',
'C' => null,
'S' => null,
);
$gender = null;
$number = null;
$person = null;
$tense = null;
$mood = null;
$subcats = str_split($subcat);
foreach ($subcats as $subcat) {
if (isset($genderNames[$subcat])) {
$gender = $em->getRepository('LexiconBundle:Gender')->findOneByValue($genderNames[$subcat]);
} elseif (isset($numberNames[$subcat])) {
$number = $em->getRepository('LexiconBundle:Number')->findOneByValue($numberNames[$subcat]);
} elseif (isset($personNames[$subcat])) {
$person = $em->getRepository('LexiconBundle:Person')->findOneByValue($personNames[$subcat]);
} else {
if (isset($tenseNames[$subcat])) {
$tense = $em->getRepository('LexiconBundle:Tense')->findOneByValue($tenseNames[$subcat]);
}
if (isset($moodNames[$subcat])) {
$mood = $em->getRepository('LexiconBundle:Mood')->findOneByValue($moodNames[$subcat]);
}
}
}
$language = $em->getRepository('LexiconBundle:Language')->find(2);
$criteria = array_filter([
'lemma' => $lemma,
'gender' => $gender,
'number' => $number,
'person' => $person,
'tense' => $tense,
'mood' => $mood,
]);
$inflection = $em->getRepository('LexiconBundle:Inflection')->findOneBy($criteria);
if (!$inflection) {
$inflection = new Inflection();
$inflection->setLemma($lemma);
$inflection->setGender($gender);
$inflection->setNumber($number);
$inflection->setPerson($person);
$inflection->setTense($tense);
$inflection->setMood($mood);
$inflection->setLanguage($language);
$inflection->setContent($form);
$inflection->setPhonetic1('');
$inflection->setPhonetic2('');
//$inflection->setStatus('');
$inflection->setCleanedContent($inflectionManager->getCleanContent($form));
$em->persist($inflection);
$em->flush();
$output->writeln('<info> insert form:'.$form.'</info>');
} else {
$output->writeln('<info> ignore form:'.$form.'</info>');
}
}
private function handleLemma($enForm, $output)
{
$em = $this->getContainer()->get('doctrine')->getEntityManager('default');
$cat = array(
'noun' => 'CommonNoun',
'verb' => 'verb',
'adjective' => 'adjective',
'interjection' => 'interjection',
'adverb' => 'adverb',
'preposition' => 'preposition',
'preposition article' => 'preposition',
'preposition determinant' => 'preposition',
'determiner' => 'determiner',
'external' => null,
'conjunction' => 'conjunction',
'pronoun' => 'pronoun',
'prefix' => null,
'verbal adjective' => 'adjective',
'nominal adjective' => 'adjective',
'subordinating conjunction' => 'conjunction',
);
$subcat = array(
'preposition determinant' => 'determinant',
'subordinating conjunction' => 'subordination',
);
$lemmaContent = $enForm['lemma'];
$lemmaCat = $enForm['lemtype'];
$lemmaSubcatName = isset($subcat[$lemmaCat])
? $subcat[$lemmaCat]
: null;
$lemmaCatName = $cat[$lemmaCat];
$lemmaCat = $em->getRepository('LexiconBundle:Category')->findOneByValue($lemmaCatName);
$lemmaSubcat = $em->getRepository('LexiconBundle:Subcategory')->findOneByValue($lemmaSubcatName);
$language = $em->getRepository('LexiconBundle:Language')->find(2);
$lemma = $em->getRepository('LexiconBundle:Lemma')->findOneBy([
'language' => $language,
'content' => $lemmaContent,
'category' => $lemmaCat,
'subcategory' => $lemmaSubcat,
]);
if ($lemmaCat && !$lemma) {
$lemma = new Lemma();
$lemma->setLanguage($language);
$lemma->setContent($lemmaContent);
$lemma->setCategory($lemmaCat);
$lemma->setLocution(0);
//$lemma->setProcessStatus(0);
$lemma->setPhonetic1('');
$lemma->setPhonetic2('');
$lemma->setSubcategory($lemmaSubcat);
//$lemma->setStatus('');
$em->persist($lemma);
$em->flush();
$output->writeln('<info> insert lemma: '.$lemmaContent.'</info>');
} else {
$output->writeln('<info> ignore lemma: '.$lemmaContent.'</info>');
}
return $lemma;
}
}
<?php
namespace LexiconBundle\Command;
use LexiconBundle\Manager\ImportManager;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Console\Style\SymfonyStyle;
class ImportLexiconCommand extends Command
{
protected function configure()
{
$this
->setName('lexicon:import')
->setDescription('Put your files, named exactly "lexicon.tsv" & "spec.txt" in a directory (ex:/euskara) in application/data/lexicons')
->addArgument('directory', InputArgument::REQUIRED, 'name of directory in application/data/lexicons');
}
public function __construct(ImportManager $im)
{
$this->im = $im;
parent::__construct();
}
protected function execute(InputInterface $input, OutputInterface $output)
{
$pathLexiconDir = $input->getArgument('directory');
if ($pathLexiconDir) {
echo 'You passed an argument: ' . $pathLexiconDir;
$this->im->importLexicon($pathLexiconDir);
} else {
echo 'Argument missing';
}
}
}
<?php
namespace LexiconBundle\Command;
use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Console\Input\InputArgument;
class InflectionCommand extends ContainerAwareCommand
{
protected function configure()