add compose

MyFiles/testES.py

@@ -70,6 +70,8 @@ def store_record(elastic_object, index_name, record):
         print('Error in indexing data')
         print(str(ex))
 
+
+    
 def parse(u):
     title = '-'
     submit_by = '-'

MyFiles/testIDB.py

@@ -60,6 +60,26 @@ def main(host='localhost', port=8086):
     print("Drop database: " + dbname)
     client.drop_database(dbname)
 
+def inser_statistcs():
+    json_body = [
+        {
+            "measurement": "byte",
+            "tags": {
+                "network": 
+                "station": 
+                "location" "channel", "latitude", "longitude", "user", "countrycode", "client"
+            },
+            "time": "2019-05-21T23:00:00Z",
+            "fields": {
+                "Float_value": 0.64,
+                "Int_value": 3,
+                "String_value": "Text",
+                "Bool_value": True
+            }
+        }
+    ]
+    print("Write points: {0}".format(json_body))
+    client.write_points(json_body,time_precision='s')
 
 def parse_args():
     """Parse the args."""

MyFiles/testflask.py

+from flask import Flask
+from flask_restplus import Api, Resource, fields
+from werkzeug.contrib.fixers import ProxyFix
+
+app = Flask(__name__)
+app.wsgi_app = ProxyFix(app.wsgi_app)
+api = Api(app, version='1.0', title='Todo API',
+    description='A simple TODO API',
+)
+
+ns = api.namespace('todos', description='TODO operations')
+
+TODOS = {
+    'todo1': {'task': 'build an API'},
+    'todo2': {'task': '?????'},
+    'todo3': {'task': 'profit!'},
+}
+
+todo = api.model('Todo', {
+    'task': fields.String(required=True, description='The task details')
+})
+
+listed_todo = api.model('ListedTodo', {
+    'id': fields.String(required=True, description='The todo ID'),
+    'todo': fields.Nested(todo, description='The Todo')
+})
+
+
+def abort_if_todo_doesnt_exist(todo_id):
+    if todo_id not in TODOS:
+        api.abort(404, "Todo {} doesn't exist".format(todo_id))
+
+parser = api.parser()
+parser.add_argument('task', type=str, required=True, help='The task details', location='form')
+
+
+@ns.route('/<string:todo_id>')
+@api.doc(responses={404: 'Todo not found'}, params={'todo_id': 'The Todo ID'})
+class Todo(Resource):
+    '''Show a single todo item and lets you delete them'''
+    @api.doc(description='todo_id should be in {0}'.format(', '.join(TODOS.keys())))
+    @api.marshal_with(todo)
+    def get(self):
+        '''Fetch a given resource'''
+        abort_if_todo_doesnt_exist('to')
+        return TODOS[todo_id]
+
+    @api.doc(responses={204: 'Todo deleted'})
+    def delete(self, todo_id):
+        '''Delete a given resource'''
+        abort_if_todo_doesnt_exist(todo_id)
+        del TODOS[todo_id]
+        return '', 204
+
+    @api.doc(parser=parser)
+    @api.marshal_with(todo)
+    def put(self, todo_id):
+        '''Update a given resource'''
+        args = parser.parse_args()
+        task = {'task': args['task']}
+        TODOS[todo_id] = task
+        return task
+
+
+@ns.route('/')
+class TodoList(Resource):
+    '''Shows a list of all todos, and lets you POST to add new tasks'''
+    @api.marshal_list_with(listed_todo)
+    def get(self):
+        '''List all todos'''
+        return [{'id': id, 'todo': todo} for id, todo in TODOS.items()]
+
+    @api.doc(parser=parser)
+    @api.marshal_with(todo, code=201)
+    def post(self):
+        '''Create a todo'''
+        args = parser.parse_args()
+        todo_id = 'todo%d' % (len(TODOS) + 1)
+        TODOS[todo_id] = {'task': args['task']}
+        return TODOS[todo_id], 201
+
+
+if __name__ == '__main__':
+    app.run(debug=True)
\ No newline at end of file

README.md

+
 ## Specs
 
-* Nombre d'accés aux données, par protocole, par réseau et par station
-* répartitions géographique approx
-* nombre d'accés au metaddonées + répartition géographiques
-* quantité de données délivrée
+ lien vers la documenantion du projet 
+   https://wiki.osug.fr/!isterre-geodata/resif/equipe/projets/ws_statistiques_acces
+
+# Envirennement de développement
+   
+## Base de données
+
+   * La base de données est chargée depuis logstash 
+   * installation:
+       - cd compose 
+       - sudo docker-compose up
+       - Dans mes développement j'ai une BD influx db en local. j'étais obligé d'ajouté une redirection de port de 8081 : 8086(par défaut). Vous pouvez enlever cette commande. 
+    * to add entree in db: 
+         -  echo '2019-06-04T13:01:32Z KOZG6620 resif-vm38.u-ga.fr 152.77.133.150 anonymous "Wget/1.18 (linux-gnu)" Z3 A181A 00 HHZ 8193' >> io/resifDEstat.log
+     * to modifie config of logstash: 
+         -  vi pipeline/filter.conf 
+         - docker exec -it logstash  kill -SIGHUP 1
 
-# to do 
- * voir influxDB
- * 
+## webservice
+    ./start.sh (lance un envirenement virtuel et lance le WS)
 
-# http://exploreflask.com/en/latest/configuration.html 
\ No newline at end of file
+PS: used to configure my project http://exploreflask.com/en/latest/configuration.html 

compose/docker-compose.yml

+version: '3'
+services:
+  logstash:
+    build: ./logstash
+    container_name: logstash
+    volumes:
+      - ./logstash/pipeline:/usr/share/logstash/pipeline
+      - ./logstash/io:/usr/share/logstash/io
+    networks:
+      - mynet
+  influxdb:
+    image: influxdb:latest
+    container_name: influxdb
+    ports:
+      - "8081:8086" 
+    networks:
+      - mynet
+networks:
+  mynet:
\ No newline at end of file

compose/logstash/Dockerfile

+FROM docker.elastic.co/logstash/logstash-oss:7.1.1
+RUN logstash-plugin install logstash-output-influxdb

compose/logstash/io/resifDEstat.log

+2019-06-04T13:01:32Z KOZG6620 resif-vm38.u-ga.fr 152.77.133.150 helmstettera "Wget/1.18 (linux-gnu)" Z3 A181A 00 HHZ 8195
+2019-06-04T13:01:32Z KOZG6620 resif-vm38.u-ga.fr 152.77.133.150 helmstettera "Wget/1.18 (linux-gnu)" Z3 A181A 00 HHZ 8195

compose/logstash/pipeline/filter.conf

+input {
+    file {
+        path => "/usr/share/logstash/io/resifDEstat.log"
+        id => "datashipment"
+        type => "datashipment"
+    }
+}
+filter {
+    if [type] == "datashipment" {
+        grok {
+            match => {
+                "message" => "(?<logdate>%{DATE}\T%{TIME}Z) %{WORD:id} %{HOSTNAME:hostname} %{IP:clientip} %{WORD:user} \"%{DATA:User_Agent}\" %{WORD:network} %{WORD:station} (%{INT:location}|%{DATA:location}) %{WORD:channel} %{NUMBER:bytes}"
+            }
+        }
+        geoip {
+            source => "clientip"
+        }
+        mutate {
+            convert => ["bytes", "integer"]
+        }
+        fingerprint {
+	    method => "MD5"
+            source => "clientip" 
+            target => "client"
+        }
+	if [user] != "anonymous" {
+        fingerprint {
+	    method => "MD5"
+            source => "user" 
+            target => "user"
+        }
+        }
+    }
+}
+output {
+    stdout {}
+    influxdb {
+        host => "influxdb"
+        db   => "statistics"
+        ssl  => false
+        measurement => "byte"
+        coerce_values => {
+            "bytes" => "integer"
+        }
+        data_points => {
+            "time" => "%{[logdate]}"
+            "bytes" => "%{[bytes]}"
+            "network" => "%{[network]}"
+            "station" => "%{[station]}"
+            "location" => "%{[location]}"
+            "channel" => "%{[channel]}"
+            "latitude" => "%{[geoip][latitude]}"
+            "longitude" => "%{[geoip][longitude]}"
+            "countrycode" => "%{[geoip][country_code3]}"
+            "user" => "%{[user]}"
+            "client" => "%{[client]}"
+            "requestid" => "%{[id]}" # est ce que ça servira ?
+        }
+        send_as_tags => ["network", "station", "location", "channel", "latitude", "longitude", "user", "countrycode", "client"]
+    }
+}

config/default.py

-default.py
\ No newline at end of file
+DEBUG = False
\ No newline at end of file

config/development.py

-#export APP_CONFIG_FILE="/home/rima/Project/RESIF/ws-statistics/config"
\ No newline at end of file
+#export APP_CONFIG_FILE="/home/rima/Project/RESIF/ws-statistics/config"
+ENV = 'development'
+DEBUG = True
\ No newline at end of file

config/production.py

+ENV = 'production'
+DEBUG =  False
\ No newline at end of file

config/test.py

+DEBUG = True
+ENV= 'test'
\ No newline at end of file

logstash-docker/logstash/Dockerfile

+FROM docker.elastic.co/logstash/logstash-oss:7.1.1
+RUN logstash-plugin install logstash-output-influxdb

logstash-docker/logstash/io/plugins/memorize.rb

+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+#
+# This filter will look for fields from an event and record the last value
+# of them.  If any are not present, their last value will be added to the
+# event.  This is useful if you want to use data from a previous event
+# on future events (for example a time field or an id field).  This differs
+# frome the multiline filter where you are combining multiple lines to
+# create a single event.
+#
+# The config looks like this:
+#
+#     filter {
+#       memorize {
+#         fields => [ "time", "id" ]
+#         default => { "time" => "00:00:00.000" }
+#       }
+#     }
+#
+# The `fields` is an array of the field NAMES that you want to memorize
+# The `default` is a map of field names to field values that you want
+# to use if the field isn't present and has no memorized value (optional)
+
+class LogStash::Filters::Memorize < LogStash::Filters::Base
+  config_name "memorize"
+  milestone 2
+
+  # An array of the field names to to memorize
+  config :fields, :validate => :array, :required => true
+  # a map for default values to use if its not seen before we need it
+  config :default, :validate => :hash, :required => false
+
+  # The stream identity is how the filter determines which stream an
+  # event belongs to. See the multiline plugin if you want more details on how
+  # this might work
+  config :stream_identity , :validate => :string, :default => "%{host}.%{path}.%{type}"
+
+  public
+  def initialize(config = {})
+    super
+
+    @threadsafe = false
+
+    # This filter needs to keep state.
+    @memorized = Hash.new
+  end # def initialize
+
+  public
+  def register
+  # nothing needed
+  end # def register
+
+  public
+  def filter(event)
+    return unless filter?(event)
+
+    any = false
+    key = event.sprintf(@stream_identity)
+    @fields.each do |field|
+      if event.get(field).nil?
+        map = @memorized[key]
+        val = map.nil? ? nil : map[field]
+        if val.nil?
+          val = @default.nil? ? nil : @default[field]
+        else
+          event.set(field, val)
+          any = true
+        end
+      else
+        map = @memorized[key]
+        if map.nil?
+          map = @memorized[key] = Hash.new
+        end
+        map[field] = event.get(field)
+      end #if
+      if any
+        filter_matched(event)
+      end
+    end #field.each
+  end
+end
+

logstash-docker/logstash/io/resifDEstat.log

+2019-06-04T13:01:32Z KOZG6620 resif-vm38.u-ga.fr 152.77.133.150 helmstettera "Wget/1.18 (linux-gnu)" Z3 A181A 00 HHZ 8192
+2019-06-04T13:01:32Z KOZG6620 resif-vm38.u-ga.fr 152.77.133.150 helmstettera "Wget/1.18 (linux-gnu)" Z3 A181A 00 HHZ 8193
+2019-06-04T13:01:32Z KOZG6620 resif-vm38.u-ga.fr 152.77.133.150 helmstettera "Wget/1.18 (linux-gnu)" Z3 A181A 00 HHZ 8194
+2019-06-04T13:01:32Z KOZG6620 resif-vm38.u-ga.fr 152.77.133.150 helmstettera "Wget/1.18 (linux-gnu)" Z3 A181A 00 HHZ 8195
+

logstash-docker/logstash/io/test.log

+152.77.156.77 - - [05/Feb/2019:10:06:07 +0000] "GET /resifws/availability/1/query?net=MT&sta=GAL HTTP/1.1" 500 287 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:60.0) Gecko/20100101 Firefox/60.0" 15998
+152.77.156.77 - - [05/Feb/2019:10:06:07 +0000] "GET /resifws/availability/1/query?net=MT&sta=GAL HTTP/1.1" 500 287 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:60.0) Gecko/20100101 Firefox/60.0" 15998
+152.77.156.77 - - [05/Feb/2019:10:06:07 +0000] "GET /resifws/availability/1/query?net=MT&sta=GAL HTTP/1.1" 500 287 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:60.0) Gecko/20100101 Firefox/60.0" 15998

logstash-docker/logstash/io/tx.log

+START CLIENT 193.52.13.61 [193.52.13.61] (SeedLink|SeedLink Client) @ 2016-11-27 00:00:01 (connected 2016-11-27 00:00:01) TX
+END CLIENT 193.52.13.61 [193.52.13.61] total TX bytes: 0
+START CLIENT 193.52.13.61 [193.52.13.61] (SeedLink|SeedLink Client) @ 2016-11-27 00:00:01 (connected 2016-11-27 00:00:01) TX
+END CLIENT 193.52.13.61 [193.52.13.61] total TX bytes: 0

logstash-docker/logstash/pipeline/filter.conf

+input {
+    file {
+        path => "/usr/share/logstash/io/resifDEstat.log"
+        id => "datashipment"
+        type => "datashipment"
+    }
+}
+filter {
+    if [type] == "datashipment" {
+        grok {
+            match => {
+                "message" => "(?<logdate>%{DATE}\T%{TIME}Z) %{WORD:id} %{HOSTNAME:hostname} %{IP:clientip} %{WORD:user} \"%{DATA:User_Agent}\" %{WORD:network} %{WORD:station} (%{INT:location}|%{DATA:location}) %{WORD:channel} %{NUMBER:bytes}"
+            }
+        }
+        geoip {
+            source => "clientip"
+        }
+        mutate {
+            convert => ["bytes", "integer"]
+        }
+        fingerprint {
+	    method => "MD5"
+            source => "clientip" 
+            target => "client"
+        }
+	if [user] != "anonymous" {
+        fingerprint {
+	    method => "MD5"
+            source => "user" 
+            target => "user"
+        }
+        }
+    }
+}
+output {
+    stdout {}
+    influxdb {
+        host => "influxdb"
+        db   => "statistics"
+        ssl  => false
+        measurement => "byte"
+        coerce_values => {
+            "bytes" => "integer"
+        }
+        data_points => {
+            "time" => "%{[logdate]}"
+            "bytes" => "%{[bytes]}"
+            "network" => "%{[network]}"
+            "station" => "%{[station]}"
+            "location" => "%{[location]}"
+            "channel" => "%{[channel]}"
+            "latitude" => "%{[geoip][latitude]}"
+            "longitude" => "%{[geoip][longitude]}"
+            "countrycode" => "%{[geoip][country_code3]}"
+            "user" => "%{[user]}"
+            "client" => "%{[client]}"
+            "requestid" => "%{[id]}" # est ce que ça servira ?
+        }
+        send_as_tags => ["network", "station", "location", "channel", "latitude", "longitude", "user", "countrycode", "client"]
+    }
+}