Commit 918987df authored by Rima Bouazzouz's avatar Rima Bouazzouz
Browse files

add compose

parent 51d4a959

Too many changes to show.

To preserve performance only 1000 of 1000+ files are displayed.
......@@ -70,6 +70,8 @@ def store_record(elastic_object, index_name, record):
print('Error in indexing data')
print(str(ex))
def parse(u):
title = '-'
submit_by = '-'
......
......@@ -60,6 +60,26 @@ def main(host='localhost', port=8086):
print("Drop database: " + dbname)
client.drop_database(dbname)
def inser_statistcs():
json_body = [
{
"measurement": "byte",
"tags": {
"network":
"station":
"location" "channel", "latitude", "longitude", "user", "countrycode", "client"
},
"time": "2019-05-21T23:00:00Z",
"fields": {
"Float_value": 0.64,
"Int_value": 3,
"String_value": "Text",
"Bool_value": True
}
}
]
print("Write points: {0}".format(json_body))
client.write_points(json_body,time_precision='s')
def parse_args():
"""Parse the args."""
......
from flask import Flask
from flask_restplus import Api, Resource, fields
from werkzeug.contrib.fixers import ProxyFix
app = Flask(__name__)
app.wsgi_app = ProxyFix(app.wsgi_app)
api = Api(app, version='1.0', title='Todo API',
description='A simple TODO API',
)
ns = api.namespace('todos', description='TODO operations')
TODOS = {
'todo1': {'task': 'build an API'},
'todo2': {'task': '?????'},
'todo3': {'task': 'profit!'},
}
todo = api.model('Todo', {
'task': fields.String(required=True, description='The task details')
})
listed_todo = api.model('ListedTodo', {
'id': fields.String(required=True, description='The todo ID'),
'todo': fields.Nested(todo, description='The Todo')
})
def abort_if_todo_doesnt_exist(todo_id):
if todo_id not in TODOS:
api.abort(404, "Todo {} doesn't exist".format(todo_id))
parser = api.parser()
parser.add_argument('task', type=str, required=True, help='The task details', location='form')
@ns.route('/<string:todo_id>')
@api.doc(responses={404: 'Todo not found'}, params={'todo_id': 'The Todo ID'})
class Todo(Resource):
'''Show a single todo item and lets you delete them'''
@api.doc(description='todo_id should be in {0}'.format(', '.join(TODOS.keys())))
@api.marshal_with(todo)
def get(self):
'''Fetch a given resource'''
abort_if_todo_doesnt_exist('to')
return TODOS[todo_id]
@api.doc(responses={204: 'Todo deleted'})
def delete(self, todo_id):
'''Delete a given resource'''
abort_if_todo_doesnt_exist(todo_id)
del TODOS[todo_id]
return '', 204
@api.doc(parser=parser)
@api.marshal_with(todo)
def put(self, todo_id):
'''Update a given resource'''
args = parser.parse_args()
task = {'task': args['task']}
TODOS[todo_id] = task
return task
@ns.route('/')
class TodoList(Resource):
'''Shows a list of all todos, and lets you POST to add new tasks'''
@api.marshal_list_with(listed_todo)
def get(self):
'''List all todos'''
return [{'id': id, 'todo': todo} for id, todo in TODOS.items()]
@api.doc(parser=parser)
@api.marshal_with(todo, code=201)
def post(self):
'''Create a todo'''
args = parser.parse_args()
todo_id = 'todo%d' % (len(TODOS) + 1)
TODOS[todo_id] = {'task': args['task']}
return TODOS[todo_id], 201
if __name__ == '__main__':
app.run(debug=True)
\ No newline at end of file
## Specs
* Nombre d'accés aux données, par protocole, par réseau et par station
* répartitions géographique approx
* nombre d'accés au metaddonées + répartition géographiques
* quantité de données délivrée
lien vers la documenantion du projet
https://wiki.osug.fr/!isterre-geodata/resif/equipe/projets/ws_statistiques_acces
# Envirennement de développement
## Base de données
* La base de données est chargée depuis logstash
* installation:
- cd compose
- sudo docker-compose up
- Dans mes développement j'ai une BD influx db en local. j'étais obligé d'ajouté une redirection de port de 8081 : 8086(par défaut). Vous pouvez enlever cette commande.
* to add entree in db:
- echo '2019-06-04T13:01:32Z KOZG6620 resif-vm38.u-ga.fr 152.77.133.150 anonymous "Wget/1.18 (linux-gnu)" Z3 A181A 00 HHZ 8193' >> io/resifDEstat.log
* to modifie config of logstash:
- vi pipeline/filter.conf
- docker exec -it logstash kill -SIGHUP 1
# to do
* voir influxDB
*
## webservice
./start.sh (lance un envirenement virtuel et lance le WS)
# http://exploreflask.com/en/latest/configuration.html
\ No newline at end of file
PS: used to configure my project http://exploreflask.com/en/latest/configuration.html
version: '3'
services:
logstash:
build: ./logstash
container_name: logstash
volumes:
- ./logstash/pipeline:/usr/share/logstash/pipeline
- ./logstash/io:/usr/share/logstash/io
networks:
- mynet
influxdb:
image: influxdb:latest
container_name: influxdb
ports:
- "8081:8086"
networks:
- mynet
networks:
mynet:
\ No newline at end of file
FROM docker.elastic.co/logstash/logstash-oss:7.1.1
RUN logstash-plugin install logstash-output-influxdb
2019-06-04T13:01:32Z KOZG6620 resif-vm38.u-ga.fr 152.77.133.150 helmstettera "Wget/1.18 (linux-gnu)" Z3 A181A 00 HHZ 8195
2019-06-04T13:01:32Z KOZG6620 resif-vm38.u-ga.fr 152.77.133.150 helmstettera "Wget/1.18 (linux-gnu)" Z3 A181A 00 HHZ 8195
input {
file {
path => "/usr/share/logstash/io/resifDEstat.log"
id => "datashipment"
type => "datashipment"
}
}
filter {
if [type] == "datashipment" {
grok {
match => {
"message" => "(?<logdate>%{DATE}\T%{TIME}Z) %{WORD:id} %{HOSTNAME:hostname} %{IP:clientip} %{WORD:user} \"%{DATA:User_Agent}\" %{WORD:network} %{WORD:station} (%{INT:location}|%{DATA:location}) %{WORD:channel} %{NUMBER:bytes}"
}
}
geoip {
source => "clientip"
}
mutate {
convert => ["bytes", "integer"]
}
fingerprint {
method => "MD5"
source => "clientip"
target => "client"
}
if [user] != "anonymous" {
fingerprint {
method => "MD5"
source => "user"
target => "user"
}
}
}
}
output {
stdout {}
influxdb {
host => "influxdb"
db => "statistics"
ssl => false
measurement => "byte"
coerce_values => {
"bytes" => "integer"
}
data_points => {
"time" => "%{[logdate]}"
"bytes" => "%{[bytes]}"
"network" => "%{[network]}"
"station" => "%{[station]}"
"location" => "%{[location]}"
"channel" => "%{[channel]}"
"latitude" => "%{[geoip][latitude]}"
"longitude" => "%{[geoip][longitude]}"
"countrycode" => "%{[geoip][country_code3]}"
"user" => "%{[user]}"
"client" => "%{[client]}"
"requestid" => "%{[id]}" # est ce que ça servira ?
}
send_as_tags => ["network", "station", "location", "channel", "latitude", "longitude", "user", "countrycode", "client"]
}
}
default.py
\ No newline at end of file
DEBUG = False
\ No newline at end of file
#export APP_CONFIG_FILE="/home/rima/Project/RESIF/ws-statistics/config"
\ No newline at end of file
#export APP_CONFIG_FILE="/home/rima/Project/RESIF/ws-statistics/config"
ENV = 'development'
DEBUG = True
\ No newline at end of file
ENV = 'production'
DEBUG = False
\ No newline at end of file
DEBUG = True
ENV= 'test'
\ No newline at end of file
FROM docker.elastic.co/logstash/logstash-oss:7.1.1
RUN logstash-plugin install logstash-output-influxdb
# encoding: utf-8
require "logstash/filters/base"
require "logstash/namespace"
#
# This filter will look for fields from an event and record the last value
# of them. If any are not present, their last value will be added to the
# event. This is useful if you want to use data from a previous event
# on future events (for example a time field or an id field). This differs
# frome the multiline filter where you are combining multiple lines to
# create a single event.
#
# The config looks like this:
#
# filter {
# memorize {
# fields => [ "time", "id" ]
# default => { "time" => "00:00:00.000" }
# }
# }
#
# The `fields` is an array of the field NAMES that you want to memorize
# The `default` is a map of field names to field values that you want
# to use if the field isn't present and has no memorized value (optional)
class LogStash::Filters::Memorize < LogStash::Filters::Base
config_name "memorize"
milestone 2
# An array of the field names to to memorize
config :fields, :validate => :array, :required => true
# a map for default values to use if its not seen before we need it
config :default, :validate => :hash, :required => false
# The stream identity is how the filter determines which stream an
# event belongs to. See the multiline plugin if you want more details on how
# this might work
config :stream_identity , :validate => :string, :default => "%{host}.%{path}.%{type}"
public
def initialize(config = {})
super
@threadsafe = false
# This filter needs to keep state.
@memorized = Hash.new
end # def initialize
public
def register
# nothing needed
end # def register
public
def filter(event)
return unless filter?(event)
any = false
key = event.sprintf(@stream_identity)
@fields.each do |field|
if event.get(field).nil?
map = @memorized[key]
val = map.nil? ? nil : map[field]
if val.nil?
val = @default.nil? ? nil : @default[field]
else
event.set(field, val)
any = true
end
else
map = @memorized[key]
if map.nil?
map = @memorized[key] = Hash.new
end
map[field] = event.get(field)
end #if
if any
filter_matched(event)
end
end #field.each
end
end
2019-06-04T13:01:32Z KOZG6620 resif-vm38.u-ga.fr 152.77.133.150 helmstettera "Wget/1.18 (linux-gnu)" Z3 A181A 00 HHZ 8192
2019-06-04T13:01:32Z KOZG6620 resif-vm38.u-ga.fr 152.77.133.150 helmstettera "Wget/1.18 (linux-gnu)" Z3 A181A 00 HHZ 8193
2019-06-04T13:01:32Z KOZG6620 resif-vm38.u-ga.fr 152.77.133.150 helmstettera "Wget/1.18 (linux-gnu)" Z3 A181A 00 HHZ 8194
2019-06-04T13:01:32Z KOZG6620 resif-vm38.u-ga.fr 152.77.133.150 helmstettera "Wget/1.18 (linux-gnu)" Z3 A181A 00 HHZ 8195
152.77.156.77 - - [05/Feb/2019:10:06:07 +0000] "GET /resifws/availability/1/query?net=MT&sta=GAL HTTP/1.1" 500 287 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:60.0) Gecko/20100101 Firefox/60.0" 15998
152.77.156.77 - - [05/Feb/2019:10:06:07 +0000] "GET /resifws/availability/1/query?net=MT&sta=GAL HTTP/1.1" 500 287 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:60.0) Gecko/20100101 Firefox/60.0" 15998
152.77.156.77 - - [05/Feb/2019:10:06:07 +0000] "GET /resifws/availability/1/query?net=MT&sta=GAL HTTP/1.1" 500 287 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:60.0) Gecko/20100101 Firefox/60.0" 15998
START CLIENT 193.52.13.61 [193.52.13.61] (SeedLink|SeedLink Client) @ 2016-11-27 00:00:01 (connected 2016-11-27 00:00:01) TX
END CLIENT 193.52.13.61 [193.52.13.61] total TX bytes: 0
START CLIENT 193.52.13.61 [193.52.13.61] (SeedLink|SeedLink Client) @ 2016-11-27 00:00:01 (connected 2016-11-27 00:00:01) TX
END CLIENT 193.52.13.61 [193.52.13.61] total TX bytes: 0
input {
file {
path => "/usr/share/logstash/io/resifDEstat.log"
id => "datashipment"
type => "datashipment"
}
}
filter {
if [type] == "datashipment" {
grok {
match => {
"message" => "(?<logdate>%{DATE}\T%{TIME}Z) %{WORD:id} %{HOSTNAME:hostname} %{IP:clientip} %{WORD:user} \"%{DATA:User_Agent}\" %{WORD:network} %{WORD:station} (%{INT:location}|%{DATA:location}) %{WORD:channel} %{NUMBER:bytes}"
}
}
geoip {
source => "clientip"
}
mutate {
convert => ["bytes", "integer"]
}
fingerprint {
method => "MD5"
source => "clientip"
target => "client"
}
if [user] != "anonymous" {
fingerprint {
method => "MD5"
source => "user"
target => "user"
}
}
}
}
output {
stdout {}
influxdb {
host => "influxdb"
db => "statistics"
ssl => false
measurement => "byte"
coerce_values => {
"bytes" => "integer"
}
data_points => {
"time" => "%{[logdate]}"
"bytes" => "%{[bytes]}"
"network" => "%{[network]}"
"station" => "%{[station]}"
"location" => "%{[location]}"
"channel" => "%{[channel]}"
"latitude" => "%{[geoip][latitude]}"
"longitude" => "%{[geoip][longitude]}"
"countrycode" => "%{[geoip][country_code3]}"
"user" => "%{[user]}"
"client" => "%{[client]}"
"requestid" => "%{[id]}" # est ce que ça servira ?
}
send_as_tags => ["network", "station", "location", "channel", "latitude", "longitude", "user", "countrycode", "client"]
}
}
from wsStatistics.app import create_app, config_logger, create_api
from flask import current_app
app = create_app()
def main():
current_app.run()
if __name__ == '__main__':
main()
source venv/bin/activate
export APP_CONFIG_FILE='/home/rima/Project/RESIF/ws-statistics/config/development.py'
python3 run.py
version: "3"
services:
logstash:
image: logstash
volumes:
- ./:/config-dir
command: logstash -f /config-dir/logstash.conf
terraform.tfstate.backup
terraform.tfstate
\ No newline at end of file
The MIT License (MIT)
Copyright (c) 2015 Nicolas Hennion
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
# Docker-compose Logstash + InfluxDB + Grafana
<img src="http://logz.io/wp-content/uploads/2015/12/logstash-logo.png" width="30%" />
## Table of Contents
- [Overview]
- [Architecture]
- [Handbooks]
Get the stack (only once):
```
git clone https://github.com/timani/docker-logstash-influxdb-grafana.git
cd docker-influxdb-grafana
docker pull logstash
docker pull grafana/grafana
docker pull influxdb
```
## Quickstart
### Logstash
If you want to run Logstash, edit the logstash.conf to yours needs and:
```
docker exec telegraf telegraf
```
### InfluxDB
If you want to run Logstash, edit the logstash.conf to yours needs and:
```
docker exec telegraf telegraf
```
### Grafana
Run your stack:
```
docker-compose up -d
```
Show me the logs:
```
docker-compose logs
```
Stop it:
```
docker-compose stop
docker-compose rm
```
Update it:
```
git pull
docker pull logstash
docker pull grafana/grafana
docker pull influxdb
```
[Overview]: https://github.com/pivotal-gss/pcf-guide#overview
[Architecture]: https://github.com/pivotal-gss/pcf-guide#architecture
[Handbooks]: https://github.com/pivotal-gss/pcf-guide#handbooks
.PHONY: all reload logs clean
all: plan apply
reload:
docker-compose exec -d logstash-grafana logstash --config.reload.automatic -f /etc/logstash/conf.d/
logs:
docker-compose logs logstash-grafana
clean:
docker-compose stop
docker-compose rm -f
docker-compose up -d
\ No newline at end of file