add compose

MyFiles/testES.py

@@ -70,6 +70,8 @@ def store_record(elastic_object, index_name, record):
         print('Error in indexing data')
         print(str(ex))
 
+
+    
 def parse(u):
     title = '-'
     submit_by = '-'

MyFiles/testIDB.py

@@ -60,6 +60,26 @@ def main(host='localhost', port=8086):
     print("Drop database: " + dbname)
     client.drop_database(dbname)
 
+def inser_statistcs():
+    json_body = [
+        {
+            "measurement": "byte",
+            "tags": {
+                "network": 
+                "station": 
+                "location" "channel", "latitude", "longitude", "user", "countrycode", "client"
+            },
+            "time": "2019-05-21T23:00:00Z",
+            "fields": {
+                "Float_value": 0.64,
+                "Int_value": 3,
+                "String_value": "Text",
+                "Bool_value": True
+            }
+        }
+    ]
+    print("Write points: {0}".format(json_body))
+    client.write_points(json_body,time_precision='s')
 
 def parse_args():
     """Parse the args."""

MyFiles/testflask.py

+from flask import Flask
+from flask_restplus import Api, Resource, fields
+from werkzeug.contrib.fixers import ProxyFix
+
+app = Flask(__name__)
+app.wsgi_app = ProxyFix(app.wsgi_app)
+api = Api(app, version='1.0', title='Todo API',
+    description='A simple TODO API',
+)
+
+ns = api.namespace('todos', description='TODO operations')
+
+TODOS = {
+    'todo1': {'task': 'build an API'},
+    'todo2': {'task': '?????'},
+    'todo3': {'task': 'profit!'},
+}
+
+todo = api.model('Todo', {
+    'task': fields.String(required=True, description='The task details')
+})
+
+listed_todo = api.model('ListedTodo', {
+    'id': fields.String(required=True, description='The todo ID'),
+    'todo': fields.Nested(todo, description='The Todo')
+})
+
+
+def abort_if_todo_doesnt_exist(todo_id):
+    if todo_id not in TODOS:
+        api.abort(404, "Todo {} doesn't exist".format(todo_id))
+
+parser = api.parser()
+parser.add_argument('task', type=str, required=True, help='The task details', location='form')
+
+
+@ns.route('/<string:todo_id>')
+@api.doc(responses={404: 'Todo not found'}, params={'todo_id': 'The Todo ID'})
+class Todo(Resource):
+    '''Show a single todo item and lets you delete them'''
+    @api.doc(description='todo_id should be in {0}'.format(', '.join(TODOS.keys())))
+    @api.marshal_with(todo)
+    def get(self):
+        '''Fetch a given resource'''
+        abort_if_todo_doesnt_exist('to')
+        return TODOS[todo_id]
+
+    @api.doc(responses={204: 'Todo deleted'})
+    def delete(self, todo_id):
+        '''Delete a given resource'''
+        abort_if_todo_doesnt_exist(todo_id)
+        del TODOS[todo_id]
+        return '', 204
+
+    @api.doc(parser=parser)
+    @api.marshal_with(todo)
+    def put(self, todo_id):
+        '''Update a given resource'''
+        args = parser.parse_args()
+        task = {'task': args['task']}
+        TODOS[todo_id] = task
+        return task
+
+
+@ns.route('/')
+class TodoList(Resource):
+    '''Shows a list of all todos, and lets you POST to add new tasks'''
+    @api.marshal_list_with(listed_todo)
+    def get(self):
+        '''List all todos'''
+        return [{'id': id, 'todo': todo} for id, todo in TODOS.items()]
+
+    @api.doc(parser=parser)
+    @api.marshal_with(todo, code=201)
+    def post(self):
+        '''Create a todo'''
+        args = parser.parse_args()
+        todo_id = 'todo%d' % (len(TODOS) + 1)
+        TODOS[todo_id] = {'task': args['task']}
+        return TODOS[todo_id], 201
+
+
+if __name__ == '__main__':
+    app.run(debug=True)
\ No newline at end of file

README.md

+
 ## Specs
 
-* Nombre d'accés aux données, par protocole, par réseau et par station
-* répartitions géographique approx
-* nombre d'accés au metaddonées + répartition géographiques
-* quantité de données délivrée
+ lien vers la documenantion du projet 
+   https://wiki.osug.fr/!isterre-geodata/resif/equipe/projets/ws_statistiques_acces
+
+# Envirennement de développement
+   
+## Base de données
+
+   * La base de données est chargée depuis logstash 
+   * installation:
+       - cd compose 
+       - sudo docker-compose up
+       - Dans mes développement j'ai une BD influx db en local. j'étais obligé d'ajouté une redirection de port de 8081 : 8086(par défaut). Vous pouvez enlever cette commande. 
+    * to add entree in db: 
+         -  echo '2019-06-04T13:01:32Z KOZG6620 resif-vm38.u-ga.fr 152.77.133.150 anonymous "Wget/1.18 (linux-gnu)" Z3 A181A 00 HHZ 8193' >> io/resifDEstat.log
+     * to modifie config of logstash: 
+         -  vi pipeline/filter.conf 
+         - docker exec -it logstash  kill -SIGHUP 1
 
-# to do 
- * voir influxDB
- * 
+## webservice
+    ./start.sh (lance un envirenement virtuel et lance le WS)
 
-# http://exploreflask.com/en/latest/configuration.html 
\ No newline at end of file
+PS: used to configure my project http://exploreflask.com/en/latest/configuration.html 

compose/docker-compose.yml

+version: '3'
+services:
+  logstash:
+    build: ./logstash
+    container_name: logstash
+    volumes:
+      - ./logstash/pipeline:/usr/share/logstash/pipeline
+      - ./logstash/io:/usr/share/logstash/io
+    networks:
+      - mynet
+  influxdb:
+    image: influxdb:latest
+    container_name: influxdb
+    ports:
+      - "8081:8086" 
+    networks:
+      - mynet
+networks:
+  mynet:
\ No newline at end of file

compose/logstash/Dockerfile

+FROM docker.elastic.co/logstash/logstash-oss:7.1.1
+RUN logstash-plugin install logstash-output-influxdb

compose/logstash/io/resifDEstat.log

+2019-06-04T13:01:32Z KOZG6620 resif-vm38.u-ga.fr 152.77.133.150 helmstettera "Wget/1.18 (linux-gnu)" Z3 A181A 00 HHZ 8195
+2019-06-04T13:01:32Z KOZG6620 resif-vm38.u-ga.fr 152.77.133.150 helmstettera "Wget/1.18 (linux-gnu)" Z3 A181A 00 HHZ 8195

compose/logstash/pipeline/filter.conf

+input {
+    file {
+        path => "/usr/share/logstash/io/resifDEstat.log"
+        id => "datashipment"
+        type => "datashipment"
+    }
+}
+filter {
+    if [type] == "datashipment" {
+        grok {
+            match => {
+                "message" => "(?<logdate>%{DATE}\T%{TIME}Z) %{WORD:id} %{HOSTNAME:hostname} %{IP:clientip} %{WORD:user} \"%{DATA:User_Agent}\" %{WORD:network} %{WORD:station} (%{INT:location}|%{DATA:location}) %{WORD:channel} %{NUMBER:bytes}"
+            }
+        }
+        geoip {
+            source => "clientip"
+        }
+        mutate {
+            convert => ["bytes", "integer"]
+        }
+        fingerprint {
+	    method => "MD5"
+            source => "clientip" 
+            target => "client"
+        }
+	if [user] != "anonymous" {
+        fingerprint {
+	    method => "MD5"
+            source => "user" 
+            target => "user"
+        }
+        }
+    }
+}
+output {
+    stdout {}
+    influxdb {
+        host => "influxdb"
+        db   => "statistics"
+        ssl  => false
+        measurement => "byte"
+        coerce_values => {
+            "bytes" => "integer"
+        }
+        data_points => {
+            "time" => "%{[logdate]}"
+            "bytes" => "%{[bytes]}"
+            "network" => "%{[network]}"
+            "station" => "%{[station]}"
+            "location" => "%{[location]}"
+            "channel" => "%{[channel]}"
+            "latitude" => "%{[geoip][latitude]}"
+            "longitude" => "%{[geoip][longitude]}"
+            "countrycode" => "%{[geoip][country_code3]}"
+            "user" => "%{[user]}"
+            "client" => "%{[client]}"
+            "requestid" => "%{[id]}" # est ce que ça servira ?
+        }
+        send_as_tags => ["network", "station", "location", "channel", "latitude", "longitude", "user", "countrycode", "client"]
+    }
+}

config/default.py

-default.py
\ No newline at end of file
+DEBUG = False
\ No newline at end of file

config/development.py

-#export APP_CONFIG_FILE="/home/rima/Project/RESIF/ws-statistics/config"
\ No newline at end of file
+#export APP_CONFIG_FILE="/home/rima/Project/RESIF/ws-statistics/config"
+ENV = 'development'
+DEBUG = True
\ No newline at end of file

config/production.py

+ENV = 'production'
+DEBUG =  False
\ No newline at end of file

config/test.py

+DEBUG = True
+ENV= 'test'
\ No newline at end of file

logstash-docker/logstash/Dockerfile

+FROM docker.elastic.co/logstash/logstash-oss:7.1.1
+RUN logstash-plugin install logstash-output-influxdb

logstash-docker/logstash/io/plugins/memorize.rb

+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+#
+# This filter will look for fields from an event and record the last value
+# of them.  If any are not present, their last value will be added to the
+# event.  This is useful if you want to use data from a previous event
+# on future events (for example a time field or an id field).  This differs
+# frome the multiline filter where you are combining multiple lines to
+# create a single event.
+#
+# The config looks like this:
+#
+#     filter {
+#       memorize {
+#         fields => [ "time", "id" ]
+#         default => { "time" => "00:00:00.000" }
+#       }
+#     }
+#
+# The `fields` is an array of the field NAMES that you want to memorize
+# The `default` is a map of field names to field values that you want
+# to use if the field isn't present and has no memorized value (optional)
+
+class LogStash::Filters::Memorize < LogStash::Filters::Base
+  config_name "memorize"
+  milestone 2
+
+  # An array of the field names to to memorize
+  config :fields, :validate => :array, :required => true
+  # a map for default values to use if its not seen before we need it
+  config :default, :validate => :hash, :required => false
+
+  # The stream identity is how the filter determines which stream an
+  # event belongs to. See the multiline plugin if you want more details on how
+  # this might work
+  config :stream_identity , :validate => :string, :default => "%{host}.%{path}.%{type}"
+
+  public
+  def initialize(config = {})
+    super
+
+    @threadsafe = false
+
+    # This filter needs to keep state.
+    @memorized = Hash.new
+  end # def initialize
+
+  public
+  def register
+  # nothing needed
+  end # def register
+
+  public
+  def filter(event)
+    return unless filter?(event)
+
+    any = false
+    key = event.sprintf(@stream_identity)
+    @fields.each do |field|
+      if event.get(field).nil?
+        map = @memorized[key]
+        val = map.nil? ? nil : map[field]
+        if val.nil?
+          val = @default.nil? ? nil : @default[field]
+        else
+          event.set(field, val)
+          any = true
+        end
+      else
+        map = @memorized[key]
+        if map.nil?
+          map = @memorized[key] = Hash.new
+        end
+        map[field] = event.get(field)
+      end #if
+      if any
+        filter_matched(event)
+      end
+    end #field.each
+  end
+end
+

logstash-docker/logstash/io/resifDEstat.log

+2019-06-04T13:01:32Z KOZG6620 resif-vm38.u-ga.fr 152.77.133.150 helmstettera "Wget/1.18 (linux-gnu)" Z3 A181A 00 HHZ 8192
+2019-06-04T13:01:32Z KOZG6620 resif-vm38.u-ga.fr 152.77.133.150 helmstettera "Wget/1.18 (linux-gnu)" Z3 A181A 00 HHZ 8193
+2019-06-04T13:01:32Z KOZG6620 resif-vm38.u-ga.fr 152.77.133.150 helmstettera "Wget/1.18 (linux-gnu)" Z3 A181A 00 HHZ 8194
+2019-06-04T13:01:32Z KOZG6620 resif-vm38.u-ga.fr 152.77.133.150 helmstettera "Wget/1.18 (linux-gnu)" Z3 A181A 00 HHZ 8195
+

logstash-docker/logstash/io/test.log

+152.77.156.77 - - [05/Feb/2019:10:06:07 +0000] "GET /resifws/availability/1/query?net=MT&sta=GAL HTTP/1.1" 500 287 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:60.0) Gecko/20100101 Firefox/60.0" 15998
+152.77.156.77 - - [05/Feb/2019:10:06:07 +0000] "GET /resifws/availability/1/query?net=MT&sta=GAL HTTP/1.1" 500 287 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:60.0) Gecko/20100101 Firefox/60.0" 15998
+152.77.156.77 - - [05/Feb/2019:10:06:07 +0000] "GET /resifws/availability/1/query?net=MT&sta=GAL HTTP/1.1" 500 287 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:60.0) Gecko/20100101 Firefox/60.0" 15998

logstash-docker/logstash/io/tx.log

+START CLIENT 193.52.13.61 [193.52.13.61] (SeedLink|SeedLink Client) @ 2016-11-27 00:00:01 (connected 2016-11-27 00:00:01) TX
+END CLIENT 193.52.13.61 [193.52.13.61] total TX bytes: 0
+START CLIENT 193.52.13.61 [193.52.13.61] (SeedLink|SeedLink Client) @ 2016-11-27 00:00:01 (connected 2016-11-27 00:00:01) TX
+END CLIENT 193.52.13.61 [193.52.13.61] total TX bytes: 0

logstash-docker/logstash/pipeline/filter.conf

+input {
+    file {
+        path => "/usr/share/logstash/io/resifDEstat.log"
+        id => "datashipment"
+        type => "datashipment"
+    }
+}
+filter {
+    if [type] == "datashipment" {
+        grok {
+            match => {
+                "message" => "(?<logdate>%{DATE}\T%{TIME}Z) %{WORD:id} %{HOSTNAME:hostname} %{IP:clientip} %{WORD:user} \"%{DATA:User_Agent}\" %{WORD:network} %{WORD:station} (%{INT:location}|%{DATA:location}) %{WORD:channel} %{NUMBER:bytes}"
+            }
+        }
+        geoip {
+            source => "clientip"
+        }
+        mutate {
+            convert => ["bytes", "integer"]
+        }
+        fingerprint {
+	    method => "MD5"
+            source => "clientip" 
+            target => "client"
+        }
+	if [user] != "anonymous" {
+        fingerprint {
+	    method => "MD5"
+            source => "user" 
+            target => "user"
+        }
+        }
+    }
+}
+output {
+    stdout {}
+    influxdb {
+        host => "influxdb"
+        db   => "statistics"
+        ssl  => false
+        measurement => "byte"
+        coerce_values => {
+            "bytes" => "integer"
+        }
+        data_points => {
+            "time" => "%{[logdate]}"
+            "bytes" => "%{[bytes]}"
+            "network" => "%{[network]}"
+            "station" => "%{[station]}"
+            "location" => "%{[location]}"
+            "channel" => "%{[channel]}"
+            "latitude" => "%{[geoip][latitude]}"
+            "longitude" => "%{[geoip][longitude]}"
+            "countrycode" => "%{[geoip][country_code3]}"
+            "user" => "%{[user]}"
+            "client" => "%{[client]}"
+            "requestid" => "%{[id]}" # est ce que ça servira ?
+        }
+        send_as_tags => ["network", "station", "location", "channel", "latitude", "longitude", "user", "countrycode", "client"]
+    }
+}

run.py

+from wsStatistics.app import create_app, config_logger,  create_api
+from flask import current_app
+
+app  = create_app()
+
+def main():
+	current_app.run()
+
+if __name__ == '__main__':
+	main()

start.sh

+
+
+source venv/bin/activate
+export APP_CONFIG_FILE='/home/rima/Project/RESIF/ws-statistics/config/development.py'
+python3 run.py

test-compose/docker-compose.yml

+version: "3"
+services:
+  logstash:
+    image: logstash
+    volumes:
+    - ./:/config-dir
+    command: logstash -f /config-dir/logstash.conf
+    

test-compose/docker-logstash-influxdb-grafana/.gitignore

+terraform.tfstate.backup
+terraform.tfstate
\ No newline at end of file

test-compose/docker-logstash-influxdb-grafana/LICENSE

+The MIT License (MIT)
+
+Copyright (c) 2015 Nicolas Hennion
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

test-compose/docker-logstash-influxdb-grafana/README.md

+# Docker-compose Logstash + InfluxDB + Grafana
+
+<img src="http://logz.io/wp-content/uploads/2015/12/logstash-logo.png" width="30%" />
+
+## Table of Contents
+
+-   [Overview]
+-   [Architecture]
+-   [Handbooks]
+
+Get the stack (only once):
+
+```
+git clone https://github.com/timani/docker-logstash-influxdb-grafana.git
+cd docker-influxdb-grafana
+docker pull logstash
+docker pull grafana/grafana
+docker pull influxdb
+```
+
+## Quickstart
+
+### Logstash
+
+If you want to run Logstash, edit the logstash.conf to yours needs and:
+
+```
+docker exec telegraf telegraf
+```
+
+### InfluxDB
+
+If you want to run Logstash, edit the logstash.conf to yours needs and:
+
+```
+docker exec telegraf telegraf
+```
+
+### Grafana 
+
+Run your stack:
+
+```
+docker-compose up -d
+
+```
+
+Show me the logs:
+
+```
+docker-compose logs
+```
+
+Stop it:
+
+```
+docker-compose stop
+docker-compose rm
+```
+
+Update it:
+
+```
+git pull
+docker pull logstash
+docker pull grafana/grafana
+docker pull influxdb
+```
+
+
+[Overview]: https://github.com/pivotal-gss/pcf-guide#overview
+[Architecture]: https://github.com/pivotal-gss/pcf-guide#architecture
+[Handbooks]: https://github.com/pivotal-gss/pcf-guide#handbooks

test-compose/docker-logstash-influxdb-grafana/docs/part-1-logstash/Makefile

+.PHONY: all reload logs clean
+
+all: plan apply
+
+reload:
+	docker-compose exec -d logstash-grafana logstash --config.reload.automatic -f /etc/logstash/conf.d/
+
+logs:
+	docker-compose logs logstash-grafana 
+
+clean:
+	docker-compose stop
+	docker-compose rm -f
+	docker-compose up -d 
\ No newline at end of file

test-compose/docker-logstash-influxdb-grafana/docs/part-1-logstash/README.md

+

test-compose/docker-logstash-influxdb-grafana/docs/part-1-logstash/docker-compose.yml

+version: '2'
+
+services:
+  logstash-grafana: