Vous avez reçu un message "Your GitLab account has been locked ..." ? Pas d'inquiétude : lisez cet article https://docs.gricad-pages.univ-grenoble-alpes.fr/help/unlock/

Commit 1a4fac89 authored by lavocat's avatar lavocat
Browse files

Move common files to Yggdrasil #12

To allow the creation of a new connector, we need to extract every
uncesseray files/code from the isengard folder. This way, creating a new
connector will be to provide an other isengard_like folder with only
specif stuff.
parent 054e9122
import pytest
from yggdrasil.isengard.isengardc import Helper
from yggdrasil import Helper
def test_helper():
helper = Helper()
......
from .log import configure_logger
from .consts import *
from .consts import bcolors
from .host_name_parser import Helper
from .event import Event
import sys
import struct
from ..isengard import consts
from .. import consts
class MPIDecoder:
......
......@@ -15,10 +15,10 @@ import queue
import multiprocessing
from threading import Lock
from threading import RLock
from ..isengard import consts
from ..isengard import Helper
from .network import Network
from ..isengard.consts import bcolors
from .. import consts
from .. import Helper
from .network import Network
from ..consts import bcolors
from .mpibridge import MPIHook
logger = logging.getLogger('yggdrasil')
......
......@@ -2,7 +2,7 @@ import time
import json
import sys
import os.path
from ..isengard import consts
from .. import consts
class FrameworkControler:
......
......@@ -5,7 +5,7 @@ import sys
import signal
import yggdrasil
from yggdrasil.erebor import Erebor
from ..isengard.log import configure_logger
from ..log import configure_logger
logger = logging.getLogger('yggdrasil')
......
import logging
import getopt
import sys
import json
import zmq
from ..isengard import consts
from .encoder import MPIDecoder
from threading import Thread
from ..isengard.socket_bridge import Bridge
from .. import consts
from .encoder import MPIDecoder
from threading import Thread
from .. import Event
logger = logging.getLogger('yggdrasil')
class Event :
def __init__(self, f, data) :
self.function = f
self.data = data
def execute(self) :
self.function(self.data)
def __str__(self) :
return "[{} data {}]".format(self.function, self.data);
class MPIHook(Thread) :
def __init__(self, add_to_queue, trans_hook, mpi_trans_hook, port, network,
......
......@@ -5,9 +5,9 @@ import time
import re
import signal
import json
from ..isengard import consts
from .. import consts
from ..isengard.isengardc import Isengard
from ..isengard.consts import bcolors
from ..consts import bcolors
logger = logging.getLogger('yggdrasil')
......
class Event:
def __init__(self, f, data):
self.function = f
self.data = data
def execute(self):
self.function(self.data)
def __str__(self):
return "[{} data {}]".format(self.function, self.data)
"""
HOSTNAMES SPECIFICATION
Hostnames given to TakTuk might be simple machine name or complex hosts
lists specifications. In its general form, an hostname is made of an host
set and an optional exclusion set separated by a slash. Each of those sets
is a comma separated list of host templates. Each of these templates is
made of constant part (characters outside brackets) and optional range
parts (characters inside brackets). Each range part is a comma separated
list of intervals or single values. Each interval is made of two single
values separated by a dash. This is true for all hostnames given to TakTuk
(both
with -m or -f options).
In other words, the following expressions are valid host
specifications:
node0
node[19]
node[1-3]
node[1-3],otherhost/node2
node[1-3,5]part[a-b]/node[3-5]parta,node1partb
they respectively expand to:
node1
node19
node1 node2 node3
node1 node3 otherhost
node1parta node2parta node2partb node3partb node5partb
Notice that these list of values are not regular expressions ("node[19]" is
"node19" and not "node1, node2, ...., node9"). Intervals are implemented
using the perl magical auto increment feature, thus you can use
alphanumeric values as interval bounds (see perl documentation, operator ++
for limitations of this auto increment).
"""
class Helper:
def build_list(self, node_list):
parts = node_list.split("/")
# construct to remove list
remove_list = []
if len(parts) > 1:
remove_list = self.extract_node_list(parts[1])
# construct to keep list
node_list = self.extract_node_list(parts[0])
for node in remove_list:
if node in node_list:
node_list.remove(node)
return node_list
def extract_node_list(self, nodes):
parts = []
ret = []
a = False
word = ""
for l in nodes:
if l == "," and not a:
parts.append(str(word))
word = ""
elif l == "[":
word = word+l
a = True
elif l == "]":
word = word+l
a = False
else:
word = word+l
parts.append(word)
for part in parts:
sub_parts = part.split("]")
sub_part = sub_parts[0]
possibilities = self.build_possibilities(sub_part)
other_parts = sub_parts[1:]
for other_part in other_parts:
to_expend = self.build_possibilities(other_part)
new_possibilities = self.expend_possibilities(possibilities,
to_expend)
possibilities = new_possibilities
ret.extend(possibilities)
return ret
def expend_possibilities(self, old, add):
ret = []
for o in old:
for a in add:
ret.append(o+a)
return ret
def build_possibilities(self, part):
possibilities = []
begin = part.find("[")
if begin > -1:
prefix = part[:begin]
to_adds = part[begin+1:].split(",")
for to_add in to_adds:
tiret = to_add.find("-")
if tiret > -1:
intervals = self.build_intervals(to_add.split("-"))
for interval in intervals:
possibilities.append(prefix+interval)
else:
possibilities.append(prefix+to_add)
else:
possibilities.append(part)
return possibilities
def build_intervals(self, interval):
intervals = []
begin = interval[0]
end = interval[1]
try:
b = int(begin)
e = int(end)
for i in range(b, e+1):
intervals.append(str(i))
except ValueError:
b = ord(begin)
e = ord(end)
for i in range(b, e+1):
intervals.append(chr(i))
return intervals
def compose(self, rank_list):
ranges = []
first = -1
last = -1
for i in range(0, len(rank_list)):
rank = rank_list[i]
if first == -1:
first = rank
last = rank
if i == len(rank_list) -1:
ranges.append("{}".format(first))
elif rank == last+1:
last = rank
if i == len(rank_list) -1:
ranges.append("{}-{}".format(first,last))
else:
if last != first:
ranges.append("{}-{}".format(first,last))
else:
ranges.append("{}".format(first))
first = rank
last = rank
if i == len(rank_list) -1:
ranges.append("{}".format(first))
return ranges
from .log import configure_logger
from .consts import *
from .consts import bcolors
from .isengardc import Isengard
from .isengardc import Helper
from .isengardc import Event
from .unix_socket_wrapper import Unix_socket
from .taktuk_wrapper import Wrapper
......@@ -3,16 +3,15 @@ import json
import time
import socket
import re
import sys
import queue
import multiprocessing
from threading import Lock
from threading import Timer
from . import consts
from .. import consts
from .unix_socket_wrapper import Unix_socket
from .socket_bridge import Bridge
from .taktuk_wrapper import Wrapper
from .consts import bcolors
from ..consts import bcolors
from .. import Helper
from .. import Event
logger = logging.getLogger('yggdrasil')
......@@ -49,153 +48,6 @@ Destination_not_available_anymore = 21
Wait_complete = 22
Wait_reduce_complete = 23
"""
HOSTNAMES SPECIFICATION
Hostnames given to TakTuk might be simple machine name or complex hosts
lists specifications. In its general form, an hostname is made of an host
set and an optional exclusion set separated by a slash. Each of those sets
is a comma separated list of host templates. Each of these templates is made
of constant part (characters outside brackets) and optional range parts
(characters inside brackets). Each range part is a comma separated list of
intervals or single values. Each interval is made of two single values
separated by a dash. This is true for all hostnames given to TakTuk (both
with -m or -f options).
In other words, the following expressions are valid host
specifications:
node0
node[19]
node[1-3]
node[1-3],otherhost/node2
node[1-3,5]part[a-b]/node[3-5]parta,node1partb
they respectively expand to:
node1
node19
node1 node2 node3
node1 node3 otherhost
node1parta node2parta node2partb node3partb node5partb
Notice that these list of values are not regular expressions ("node[19]" is
"node19" and not "node1, node2, ...., node9"). Intervals are implemented
using the perl magical auto increment feature, thus you can use alphanumeric
values as interval bounds (see perl documentation, operator ++ for
limitations of this auto increment).
"""
class Helper:
def build_list(self, node_list):
parts = node_list.split("/")
# construct to remove list
remove_list = []
if len(parts) > 1 :
remove_list = self.extract_node_list(parts[1])
# construct to keep list
node_list = self.extract_node_list(parts[0])
for node in remove_list :
if node in node_list :
node_list.remove(node)
return node_list
def extract_node_list(self, nodes) :
parts = []
ret = []
a = False
b = False
word = ""
for l in nodes :
if l == "," and not a :
parts.append(str(word))
word = ""
elif l == "[" :
word = word+l
a = True
elif l == "]" :
word = word+l
a = False
else :
word = word+l
parts.append(word)
for part in parts :
sub_parts = part.split("]")
sub_part = sub_parts[0]
possibilities = self.build_possibilities(sub_part)
other_parts = sub_parts[1:]
for other_part in other_parts :
to_expend = self.build_possibilities(other_part)
new_possibilities = self.expend_possibilities(possibilities,
to_expend)
possibilities = new_possibilities
ret.extend(possibilities)
return ret
def expend_possibilities(self, old, add) :
ret = []
for o in old :
for a in add :
ret.append(o+a)
return ret
def build_possibilities(self, part) :
possibilities = []
begin = part.find("[")
if begin > -1 :
prefix = part[:begin]
to_adds = part[begin+1:].split(",")
for to_add in to_adds :
tiret = to_add.find("-")
if tiret > -1 :
intervals = self.build_intervals(to_add.split("-"))
for interval in intervals :
possibilities.append(prefix+interval)
else :
possibilities.append(prefix+to_add)
else :
possibilities.append(part)
return possibilities
def build_intervals(self, interval) :
intervals = []
begin = interval[0]
end = interval[1]
try :
b = int(begin)
e = int(end)
for i in range(b, e+1) :
intervals.append(str(i))
except ValueError :
b = ord(begin)
e = ord(end)
for i in range(b, e+1) :
intervals.append(chr(i))
return intervals
def compose(self, rank_list) :
ranges = []
first = -1
last = -1
for i in range(0, len(rank_list)) :
rank = rank_list[i]
if first == -1 :
first = rank
last = rank
if i == len(rank_list) -1 :
ranges.append("{}".format(first))
elif rank == last+1 :
last = rank
if i == len(rank_list) -1 :
ranges.append("{}-{}".format(first,last))
else :
if last != first :
ranges.append("{}-{}".format(first,last))
else :
ranges.append("{}".format(first))
first = rank
last = rank
if i == len(rank_list) -1 :
ranges.append("{}".format(first))
return ranges
class Spawned :
def __init__(self, name):
......@@ -254,17 +106,6 @@ class Spawned :
def is_failed(self) :
return self.failed
class Event :
def __init__(self, f, data) :
self.function = f
self.data = data
def execute(self) :
self.function(self.data)
def __str__(self) :
return "[{} data {}]".format(self.function, self.data);
# Class Isengard
# Author: Thomas Lavocat
#
......
......@@ -13,8 +13,8 @@ import sys
import socket
import select
import yggdrasil
from . import configure_logger
from . import consts
from .. import configure_logger
from .. import consts
from threading import Thread
from threading import Timer
from threading import Lock
......
......@@ -2,7 +2,7 @@ import logging
import sys
import collections
import pexpect
from . import consts
from .. import consts
from threading import Thread
logger = logging.getLogger('yggdrasil')
......
......@@ -6,7 +6,7 @@ import select
import queue
import socket
import os
from . import consts
from .. import consts
from threading import Lock
from threading import Thread
......
import re
import json
from yggdrasil.isengard import consts
from .. import consts
from .message_receiver import MessageReceiver
class Barrier(MessageReceiver):
......
import base64
import json
from yggdrasil.isengard import consts
from .. import consts
from .group import Group
from .executor import Executor
......
......@@ -9,7 +9,7 @@ import signal
import threading
from yggdrasil.erebor import FrameworkControler
from yggdrasil.erebor import Erebor
from yggdrasil.isengard import consts
from .. import consts
class ComponentRuler(FrameworkControler):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment