Commit 918987df authored by Rima Bouazzouz's avatar Rima Bouazzouz
Browse files

add compose

parent 51d4a959
......@@ -70,6 +70,8 @@ def store_record(elastic_object, index_name, record):
print('Error in indexing data')
print(str(ex))
def parse(u):
title = '-'
submit_by = '-'
......
......@@ -60,6 +60,26 @@ def main(host='localhost', port=8086):
print("Drop database: " + dbname)
client.drop_database(dbname)
def inser_statistcs():
json_body = [
{
"measurement": "byte",
"tags": {
"network":
"station":
"location" "channel", "latitude", "longitude", "user", "countrycode", "client"
},
"time": "2019-05-21T23:00:00Z",
"fields": {
"Float_value": 0.64,
"Int_value": 3,
"String_value": "Text",
"Bool_value": True
}
}
]
print("Write points: {0}".format(json_body))
client.write_points(json_body,time_precision='s')
def parse_args():
"""Parse the args."""
......
from flask import Flask
from flask_restplus import Api, Resource, fields
from werkzeug.contrib.fixers import ProxyFix
app = Flask(__name__)
app.wsgi_app = ProxyFix(app.wsgi_app)
api = Api(app, version='1.0', title='Todo API',
description='A simple TODO API',
)
ns = api.namespace('todos', description='TODO operations')
TODOS = {
'todo1': {'task': 'build an API'},
'todo2': {'task': '?????'},
'todo3': {'task': 'profit!'},
}
todo = api.model('Todo', {
'task': fields.String(required=True, description='The task details')
})
listed_todo = api.model('ListedTodo', {
'id': fields.String(required=True, description='The todo ID'),
'todo': fields.Nested(todo, description='The Todo')
})
def abort_if_todo_doesnt_exist(todo_id):
if todo_id not in TODOS:
api.abort(404, "Todo {} doesn't exist".format(todo_id))
parser = api.parser()
parser.add_argument('task', type=str, required=True, help='The task details', location='form')
@ns.route('/<string:todo_id>')
@api.doc(responses={404: 'Todo not found'}, params={'todo_id': 'The Todo ID'})
class Todo(Resource):
'''Show a single todo item and lets you delete them'''
@api.doc(description='todo_id should be in {0}'.format(', '.join(TODOS.keys())))
@api.marshal_with(todo)
def get(self):
'''Fetch a given resource'''
abort_if_todo_doesnt_exist('to')
return TODOS[todo_id]
@api.doc(responses={204: 'Todo deleted'})
def delete(self, todo_id):
'''Delete a given resource'''
abort_if_todo_doesnt_exist(todo_id)
del TODOS[todo_id]
return '', 204
@api.doc(parser=parser)
@api.marshal_with(todo)
def put(self, todo_id):
'''Update a given resource'''
args = parser.parse_args()
task = {'task': args['task']}
TODOS[todo_id] = task
return task
@ns.route('/')
class TodoList(Resource):
'''Shows a list of all todos, and lets you POST to add new tasks'''
@api.marshal_list_with(listed_todo)
def get(self):
'''List all todos'''
return [{'id': id, 'todo': todo} for id, todo in TODOS.items()]
@api.doc(parser=parser)
@api.marshal_with(todo, code=201)
def post(self):
'''Create a todo'''
args = parser.parse_args()
todo_id = 'todo%d' % (len(TODOS) + 1)
TODOS[todo_id] = {'task': args['task']}
return TODOS[todo_id], 201
if __name__ == '__main__':
app.run(debug=True)
\ No newline at end of file
## Specs
* Nombre d'accés aux données, par protocole, par réseau et par station
* répartitions géographique approx
* nombre d'accés au metaddonées + répartition géographiques
* quantité de données délivrée
lien vers la documenantion du projet
https://wiki.osug.fr/!isterre-geodata/resif/equipe/projets/ws_statistiques_acces
# Envirennement de développement
## Base de données
* La base de données est chargée depuis logstash
* installation:
- cd compose
- sudo docker-compose up
- Dans mes développement j'ai une BD influx db en local. j'étais obligé d'ajouté une redirection de port de 8081 : 8086(par défaut). Vous pouvez enlever cette commande.
* to add entree in db:
- echo '2019-06-04T13:01:32Z KOZG6620 resif-vm38.u-ga.fr 152.77.133.150 anonymous "Wget/1.18 (linux-gnu)" Z3 A181A 00 HHZ 8193' >> io/resifDEstat.log
* to modifie config of logstash:
- vi pipeline/filter.conf
- docker exec -it logstash kill -SIGHUP 1
# to do
* voir influxDB
*
## webservice
./start.sh (lance un envirenement virtuel et lance le WS)
# http://exploreflask.com/en/latest/configuration.html
\ No newline at end of file
PS: used to configure my project http://exploreflask.com/en/latest/configuration.html
version: '3'
services:
logstash:
build: ./logstash
container_name: logstash
volumes:
- ./logstash/pipeline:/usr/share/logstash/pipeline
- ./logstash/io:/usr/share/logstash/io
networks:
- mynet
influxdb:
image: influxdb:latest
container_name: influxdb
ports:
- "8081:8086"
networks:
- mynet
networks:
mynet:
\ No newline at end of file
FROM docker.elastic.co/logstash/logstash-oss:7.1.1
RUN logstash-plugin install logstash-output-influxdb
2019-06-04T13:01:32Z KOZG6620 resif-vm38.u-ga.fr 152.77.133.150 helmstettera "Wget/1.18 (linux-gnu)" Z3 A181A 00 HHZ 8195
2019-06-04T13:01:32Z KOZG6620 resif-vm38.u-ga.fr 152.77.133.150 helmstettera "Wget/1.18 (linux-gnu)" Z3 A181A 00 HHZ 8195
input {
file {
path => "/usr/share/logstash/io/resifDEstat.log"
id => "datashipment"
type => "datashipment"
}
}
filter {
if [type] == "datashipment" {
grok {
match => {
"message" => "(?<logdate>%{DATE}\T%{TIME}Z) %{WORD:id} %{HOSTNAME:hostname} %{IP:clientip} %{WORD:user} \"%{DATA:User_Agent}\" %{WORD:network} %{WORD:station} (%{INT:location}|%{DATA:location}) %{WORD:channel} %{NUMBER:bytes}"
}
}
geoip {
source => "clientip"
}
mutate {
convert => ["bytes", "integer"]
}
fingerprint {
method => "MD5"
source => "clientip"
target => "client"
}
if [user] != "anonymous" {
fingerprint {
method => "MD5"
source => "user"
target => "user"
}
}
}
}
output {
stdout {}
influxdb {
host => "influxdb"
db => "statistics"
ssl => false
measurement => "byte"
coerce_values => {
"bytes" => "integer"
}
data_points => {
"time" => "%{[logdate]}"
"bytes" => "%{[bytes]}"
"network" => "%{[network]}"
"station" => "%{[station]}"
"location" => "%{[location]}"
"channel" => "%{[channel]}"
"latitude" => "%{[geoip][latitude]}"
"longitude" => "%{[geoip][longitude]}"
"countrycode" => "%{[geoip][country_code3]}"
"user" => "%{[user]}"
"client" => "%{[client]}"
"requestid" => "%{[id]}" # est ce que ça servira ?
}
send_as_tags => ["network", "station", "location", "channel", "latitude", "longitude", "user", "countrycode", "client"]
}
}
default.py
\ No newline at end of file
DEBUG = False
\ No newline at end of file
#export APP_CONFIG_FILE="/home/rima/Project/RESIF/ws-statistics/config"
\ No newline at end of file
#export APP_CONFIG_FILE="/home/rima/Project/RESIF/ws-statistics/config"
ENV = 'development'
DEBUG = True
\ No newline at end of file
ENV = 'production'
DEBUG = False
\ No newline at end of file
DEBUG = True
ENV= 'test'
\ No newline at end of file
FROM docker.elastic.co/logstash/logstash-oss:7.1.1
RUN logstash-plugin install logstash-output-influxdb
# encoding: utf-8
require "logstash/filters/base"
require "logstash/namespace"
#
# This filter will look for fields from an event and record the last value
# of them. If any are not present, their last value will be added to the
# event. This is useful if you want to use data from a previous event
# on future events (for example a time field or an id field). This differs
# frome the multiline filter where you are combining multiple lines to
# create a single event.
#
# The config looks like this:
#
# filter {
# memorize {
# fields => [ "time", "id" ]
# default => { "time" => "00:00:00.000" }
# }
# }
#
# The `fields` is an array of the field NAMES that you want to memorize
# The `default` is a map of field names to field values that you want
# to use if the field isn't present and has no memorized value (optional)
class LogStash::Filters::Memorize < LogStash::Filters::Base
config_name "memorize"
milestone 2
# An array of the field names to to memorize
config :fields, :validate => :array, :required => true
# a map for default values to use if its not seen before we need it
config :default, :validate => :hash, :required => false
# The stream identity is how the filter determines which stream an
# event belongs to. See the multiline plugin if you want more details on how
# this might work
config :stream_identity , :validate => :string, :default => "%{host}.%{path}.%{type}"
public
def initialize(config = {})
super
@threadsafe = false
# This filter needs to keep state.
@memorized = Hash.new
end # def initialize
public
def register
# nothing needed
end # def register
public
def filter(event)
return unless filter?(event)
any = false
key = event.sprintf(@stream_identity)
@fields.each do |field|
if event.get(field).nil?
map = @memorized[key]
val = map.nil? ? nil : map[field]
if val.nil?
val = @default.nil? ? nil : @default[field]
else
event.set(field, val)
any = true
end
else
map = @memorized[key]
if map.nil?
map = @memorized[key] = Hash.new
end
map[field] = event.get(field)
end #if
if any
filter_matched(event)
end
end #field.each
end
end
2019-06-04T13:01:32Z KOZG6620 resif-vm38.u-ga.fr 152.77.133.150 helmstettera "Wget/1.18 (linux-gnu)" Z3 A181A 00 HHZ 8192
2019-06-04T13:01:32Z KOZG6620 resif-vm38.u-ga.fr 152.77.133.150 helmstettera "Wget/1.18 (linux-gnu)" Z3 A181A 00 HHZ 8193
2019-06-04T13:01:32Z KOZG6620 resif-vm38.u-ga.fr 152.77.133.150 helmstettera "Wget/1.18 (linux-gnu)" Z3 A181A 00 HHZ 8194
2019-06-04T13:01:32Z KOZG6620 resif-vm38.u-ga.fr 152.77.133.150 helmstettera "Wget/1.18 (linux-gnu)" Z3 A181A 00 HHZ 8195
152.77.156.77 - - [05/Feb/2019:10:06:07 +0000] "GET /resifws/availability/1/query?net=MT&sta=GAL HTTP/1.1" 500 287 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:60.0) Gecko/20100101 Firefox/60.0" 15998
152.77.156.77 - - [05/Feb/2019:10:06:07 +0000] "GET /resifws/availability/1/query?net=MT&sta=GAL HTTP/1.1" 500 287 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:60.0) Gecko/20100101 Firefox/60.0" 15998
152.77.156.77 - - [05/Feb/2019:10:06:07 +0000] "GET /resifws/availability/1/query?net=MT&sta=GAL HTTP/1.1" 500 287 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:60.0) Gecko/20100101 Firefox/60.0" 15998
START CLIENT 193.52.13.61 [193.52.13.61] (SeedLink|SeedLink Client) @ 2016-11-27 00:00:01 (connected 2016-11-27 00:00:01) TX
END CLIENT 193.52.13.61 [193.52.13.61] total TX bytes: 0
START CLIENT 193.52.13.61 [193.52.13.61] (SeedLink|SeedLink Client) @ 2016-11-27 00:00:01 (connected 2016-11-27 00:00:01) TX
END CLIENT 193.52.13.61 [193.52.13.61] total TX bytes: 0
input {
file {
path => "/usr/share/logstash/io/resifDEstat.log"
id => "datashipment"
type => "datashipment"
}
}
filter {
if [type] == "datashipment" {
grok {
match => {
"message" => "(?<logdate>%{DATE}\T%{TIME}Z) %{WORD:id} %{HOSTNAME:hostname} %{IP:clientip} %{WORD:user} \"%{DATA:User_Agent}\" %{WORD:network} %{WORD:station} (%{INT:location}|%{DATA:location}) %{WORD:channel} %{NUMBER:bytes}"
}
}
geoip {
source => "clientip"
}
mutate {
convert => ["bytes", "integer"]
}
fingerprint {
method => "MD5"
source => "clientip"
target => "client"
}
if [user] != "anonymous" {
fingerprint {
method => "MD5"
source => "user"
target => "user"
}
}
}
}
output {
stdout {}
influxdb {
host => "influxdb"
db => "statistics"
ssl => false
measurement => "byte"
coerce_values => {
"bytes" => "integer"
}
data_points => {
"time" => "%{[logdate]}"
"bytes" => "%{[bytes]}"
"network" => "%{[network]}"
"station" => "%{[station]}"
"location" => "%{[location]}"
"channel" => "%{[channel]}"
"latitude" => "%{[geoip][latitude]}"
"longitude" => "%{[geoip][longitude]}"
"countrycode" => "%{[geoip][country_code3]}"
"user" => "%{[user]}"
"client" => "%{[client]}"
"requestid" => "%{[id]}" # est ce que ça servira ?
}
send_as_tags => ["network", "station", "location", "channel", "latitude", "longitude", "user", "countrycode", "client"]
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment