Commit 70398399 authored by Didier DONSEZ's avatar Didier DONSEZ
Browse files

initial import of reporting scripts

parent d7fe9169
node_modules/
admin.token.json
.*.json
.*.html
.DS_Store
*.bak
*.old
# Check health of the Chirpstack LNS
## Prerequises
### Linux
```bash
sudo apt install jq curl
```
### MacOS
```bash
brew install jq curl
```
## Configure
```bash
cp config.tmpl.json .config.json
cp credentials.tmpl.json .credentials.json
```
Edit `.config.json` and `.credentials.json`
Then edit the crontab with `crontab -e` and add the following line into the crontab
```
0 12 * * * /bin/bash /home/campusiot/chirpstack-monitoring/reports/cron.sh
```
## Generate reports
```bash
./generate_reports.sh
open .gateways.html
open .applications.html
open .devices.html
open .organizations.html
```
## Send reports by email
```bash
./sendmail_reports.sh
```
## Send gateway alert by email to their owners
> The owner's email address should be configured into the tag "owner_email" of the gateway description on the LNS console
```bash
./mail_to_gateways_owners.sh
open .mails.csv
```
## Useful utilities
* https://www.npmjs.com/package/csvtojson
* https://www.npmjs.com/package/jsontocsv
* https://stedolan.github.io/jq/manual/
```bash
sudo npm install -g csvtojson
csvtojson --help
sudo npm install -g jsontocsv
jsontocsv --help
```
```bash
jq '[.[] | {"name":.name, "deveui":.deveui, "appkey":.appkey}]' fulldevices.json > devices.json
```
### TODOLIST
* [x] send email to the persons (one or more) responsible of a failed gateway. responsibles are set into the tag "owner_email" into the gateway description on the LNS
* [ ] add organizationID into the device report (join with .applications.json) https://github.com/stedolan/jq/issues/1090
* [x] send reports by email
* [x] crontab for sending reports by email
* [ ] convert date to epoch for computing the time ago
** jq: error (at .gateways.json:0): date "2020-12-21T09:24:02.077751Z" does not match format "%Y-%m-%dT%H:%M:%SZ" (.lastSeenAt | fromdateiso8601)
\ No newline at end of file
#!/bin/bash
# Copyright (C) CampusIoT, - All Rights Reserved
# Written by CampusIoT Dev Team, 2016-2021
rm -f admin.token.json
rm -f admin.token.json
rm -f .organizations.*
rm -f .gateways.*
rm -f .gateway-.*
rm -f .devices.*
rm -f .applications.*
{
"report_email_to": "XX.XX@univ-grenoble-alpes.fr YY.YY@univ-grenoble-alpes.fr"
}
\ No newline at end of file
{
"username": "admin",
"password": "XXXXXXXXXXXX"
}
\ No newline at end of file
#!/bin/bash
# Copyright (C) CampusIoT, - All Rights Reserved
# Written by CampusIoT Dev Team, 2016-2021
# crontab entry
# Every day at noon
# 0 12 * * * /bin/bash /home/campusiot/chirpstack-monitoring/reports/cron.sh
cd ~/chirpstack-monitoring/reports/cron.sh
./generate_reports.sh
./sendmail_reports.sh
./mail_to_gateways_owners.sh
.result | sort_by(.lastSeenAt, .devEUI) | reverse [] | ("<li><a href='" + @uri "https://lns.campusiot.imag.fr/#/organizations/\(.organizationID)/applications/\(.applicationID)/devices/\(.devEUI)" + "'>" + .devEUI + "</a>: " + .name + " (profile " + .deviceProfileName + " ,org " + .organizationID + ") - " + .lastSeenAt + "</li>" )
.result | sort_by(.lastSeenAt, .id) | reverse [] | ("<li><a href='" + @uri "https://lns.campusiot.imag.fr/#/organizations/\(.organizationID)/gateways/\(.id)" + "'>" + .id + "</a>: " + .name + " - (org " + .organizationID + ") - " + .lastSeenAt + "</li>" )
#!/bin/bash
USERNAME=admin
PASSWORD=XXXXXXXXXXXXXXXX
JWT=$(./get_jwt.sh $USERNAME $PASSWORD)
./get_organizations.sh $JWT
./get_gateways.sh $JWT
./get_devices.sh $JWT
# TODO send reports by email
#!/bin/bash
# Copyright (C) CampusIoT, - All Rights Reserved
# Written by CampusIoT Dev Team, 2016-2021
# ------------------------------------------------
# Get applications
# ------------------------------------------------
# Parameters
if [[ $# -ne 1 ]] ; then
echo "Usage: $0 JWT"
exit 1
fi
TOKEN="$1"
AUTH="Grpc-Metadata-Authorization: Bearer $TOKEN"
#sudo npm install -g jwt-cli
#jwt $TOKEN
# Installation
if ! [ -x "$(command -v jq)" ]; then
echo 'jq is not installed. Installing jq ...'
sudo apt-get install -y jq
fi
if ! [ -x "$(command -v curl)" ]; then
echo 'curl is not installed. Installing curl ...'
sudo apt-get install -y curl
fi
# Content-Type
ACCEPT_JSON="Accept: application/json"
ACCEPT_CSV="Accept: text/csv"
CONTENT_JSON="Content-Type: application/json"
CONTENT_CSV="Content-Type: text/csv"
# LOCAL
#PORT=8888
#URL=http://localhost:$PORT
# PROD
PORT=443
URL=https://lns.campusiot.imag.fr:$PORT
# Operations
# Operations
#CURL="curl --verbose"
CURL="curl -s --insecure"
#CURL="curl -s"
GET="${CURL} -X GET --header \""$ACCEPT_JSON"\""
POST="${CURL} -X POST --header \""$ACCEPT_JSON"\""
PUT="${CURL} -X PUT --header \""$ACCEPT_JSON"\""
DELETE="${CURL} -X DELETE --header \""$ACCEPT_JSON"\""
OPTIONS="${CURL} -X OPTIONS --header \""$ACCEPT_JSON"\""
HEAD="${CURL} -X HEAD --header \""$ACCEPT_JSON"\""
${GET} \
--header "$AUTH" ${URL}'/api/applications?limit=9999&offset=0' \
> .applications.json
jq '.result[] | ( .id + ": " + .name + " - " + .organizationID + " - " + .description)' .applications.json
#!/bin/bash
# Copyright (C) CampusIoT, - All Rights Reserved
# Written by CampusIoT Dev Team, 2016-2021
# ------------------------------------------------
# Get devices
# ------------------------------------------------
# Parameters
if [[ $# -ne 1 ]] ; then
echo "Usage: $0 JWT"
exit 1
fi
TOKEN="$1"
AUTH="Grpc-Metadata-Authorization: Bearer $TOKEN"
#sudo npm install -g jwt-cli
#jwt $TOKEN
# Installation
if ! [ -x "$(command -v jq)" ]; then
echo 'jq is not installed. Installing jq ...'
sudo apt-get install -y jq
fi
if ! [ -x "$(command -v curl)" ]; then
echo 'curl is not installed. Installing curl ...'
sudo apt-get install -y curl
fi
# Content-Type
ACCEPT_JSON="Accept: application/json"
ACCEPT_CSV="Accept: text/csv"
CONTENT_JSON="Content-Type: application/json"
CONTENT_CSV="Content-Type: text/csv"
# LOCAL
#PORT=8888
#URL=http://localhost:$PORT
# PROD
PORT=443
URL=https://lns.campusiot.imag.fr:$PORT
# Operations
# Operations
#CURL="curl --verbose"
CURL="curl -s --insecure"
#CURL="curl -s"
GET="${CURL} -X GET --header \""$ACCEPT_JSON"\""
POST="${CURL} -X POST --header \""$ACCEPT_JSON"\""
PUT="${CURL} -X PUT --header \""$ACCEPT_JSON"\""
DELETE="${CURL} -X DELETE --header \""$ACCEPT_JSON"\""
OPTIONS="${CURL} -X OPTIONS --header \""$ACCEPT_JSON"\""
HEAD="${CURL} -X HEAD --header \""$ACCEPT_JSON"\""
${GET} \
--header "$AUTH" ${URL}'/api/devices?limit=9999&offset=0' \
> .devices.json
echo '<html><head><title>CampusIoT LNS :: Devices</title></head><body style="font-family:verdana;"><h1>CampusIoT LNS :: Devices</h1>' > .devices.html
TODAY=$(date +"%Y-%m-%d")
echo '<p>generated at ' >> .devices.html
date +"%Y-%m-%d %T %Z" >> .devices.html
echo ' - ' >> .devices.html
TZ=GMT date +"%Y-%m-%d %T %Z" >> .devices.html
echo '</p>' >> .devices.html
echo '<h2>Active devices</h2>' >> .devices.html
jq --raw-output -f devices_to_html.jq .devices.json | grep $TODAY >> .devices.html
echo '<h2>Passive devices</h2>' >> .devices.html
jq --raw-output -f devices_to_html.jq .devices.json | grep -v $TODAY >> .devices.html
echo '</body></html>' >> .devices.html
#!/bin/bash
# Copyright (C) CampusIoT, - All Rights Reserved
# Written by CampusIoT Dev Team, 2016-2021
# ------------------------------------------------
# Get getaways
# ------------------------------------------------
# Parameters
if [[ $# -ne 2 ]] ; then
echo "Usage: $0 JWT GWID"
exit 1
fi
TOKEN="$1"
GWID="$2"
AUTH="Grpc-Metadata-Authorization: Bearer $TOKEN"
#sudo npm install -g jwt-cli
#jwt $TOKEN
# Installation
if ! [ -x "$(command -v jq)" ]; then
echo 'jq is not installed. Installing jq ...'
sudo apt-get install -y jq
fi
if ! [ -x "$(command -v curl)" ]; then
echo 'curl is not installed. Installing curl ...'
sudo apt-get install -y curl
fi
# Content-Type
ACCEPT_JSON="Accept: application/json"
ACCEPT_CSV="Accept: text/csv"
CONTENT_JSON="Content-Type: application/json"
CONTENT_CSV="Content-Type: text/csv"
# LOCAL
#PORT=8888
#URL=http://localhost:$PORT
# PROD
PORT=443
URL=https://lns.campusiot.imag.fr:$PORT
# Operations
#CURL="curl --verbose"
CURL="curl -s --insecure"
#CURL="curl -s"
GET="${CURL} -X GET --header \""$ACCEPT_JSON"\""
POST="${CURL} -X POST --header \""$ACCEPT_JSON"\""
PUT="${CURL} -X PUT --header \""$ACCEPT_JSON"\""
DELETE="${CURL} -X DELETE --header \""$ACCEPT_JSON"\""
OPTIONS="${CURL} -X OPTIONS --header \""$ACCEPT_JSON"\""
HEAD="${CURL} -X HEAD --header \""$ACCEPT_JSON"\""
${GET} \
--header "$AUTH" ${URL}'/api/gateways/'${GWID} \
> .gateway-${GWID}.json
#!/bin/bash
# Copyright (C) CampusIoT, - All Rights Reserved
# Written by CampusIoT Dev Team, 2016-2021
# ------------------------------------------------
# Get getaways
# ------------------------------------------------
# Parameters
if [[ $# -ne 1 ]] ; then
echo "Usage: $0 JWT"
exit 1
fi
TOKEN="$1"
AUTH="Grpc-Metadata-Authorization: Bearer $TOKEN"
#sudo npm install -g jwt-cli
#jwt $TOKEN
# Installation
if ! [ -x "$(command -v jq)" ]; then
echo 'jq is not installed. Installing jq ...'
sudo apt-get install -y jq
fi
if ! [ -x "$(command -v curl)" ]; then
echo 'curl is not installed. Installing curl ...'
sudo apt-get install -y curl
fi
# Content-Type
ACCEPT_JSON="Accept: application/json"
ACCEPT_CSV="Accept: text/csv"
CONTENT_JSON="Content-Type: application/json"
CONTENT_CSV="Content-Type: text/csv"
# LOCAL
#PORT=8888
#URL=http://localhost:$PORT
# PROD
PORT=443
URL=https://lns.campusiot.imag.fr:$PORT
# Operations
#CURL="curl --verbose"
CURL="curl -s --insecure"
#CURL="curl -s"
GET="${CURL} -X GET --header \""$ACCEPT_JSON"\""
POST="${CURL} -X POST --header \""$ACCEPT_JSON"\""
PUT="${CURL} -X PUT --header \""$ACCEPT_JSON"\""
DELETE="${CURL} -X DELETE --header \""$ACCEPT_JSON"\""
OPTIONS="${CURL} -X OPTIONS --header \""$ACCEPT_JSON"\""
HEAD="${CURL} -X HEAD --header \""$ACCEPT_JSON"\""
${GET} \
--header "$AUTH" ${URL}'/api/gateways?limit=1000&offset=0' \
> .gateways.json
echo '<html><head><title>CampusIoT LNS :: Gateways</title></head><body style="font-family:verdana;"><h1>CampusIoT LNS :: Gateways</h1>' > .gateways.html
TODAY=$(date +"%Y-%m-%d")
echo '<p>generated at ' >> .gateways.html
date +"%Y-%m-%d %T %Z" >> .gateways.html
echo ' - ' >> .gateways.html
TZ=GMT date +"%Y-%m-%d %T %Z" >> .gateways.html
echo '</p>' >> .gateways.html
echo '<h2>Active gateways</h2>' >> .gateways.html
jq --raw-output -f gateways_to_html.jq .gateways.json | grep $TODAY >> .gateways.html
echo '<h2>Passive gateways</h2>' >> .gateways.html
jq --raw-output -f gateways_to_html.jq .gateways.json | grep -v $TODAY >> .gateways.html
echo '</body></html>' >> .gateways.html
GATEWAYS=$(jq --raw-output ".result | sort_by(.lastSeenAt, .id) | reverse [] | (.id)" .gateways.json)
for g in $GATEWAYS
do
echo "get details for $g"
./get_gateway.sh $TOKEN $g
done
#!/bin/bash
# Copyright (C) CampusIoT, - All Rights Reserved
# Written by CampusIoT Dev Team, 2016-2021
# Parameters
if [[ $# -ne 2 ]] ; then
echo "Usage: $0 USERNAME PASSWORD"
exit 1
fi
USERNAME=$1
PASSWORD=$2
# TODO get AUTH_JSON from the $(echo .credentials.json)
AUTH_JSON="{ \"username\": \"${USERNAME}\", \"password\": \"${PASSWORD}\" }"
# Installation
if ! [ -x "$(command -v jq)" ]; then
>&2 echo 'jq is not installed. Installing jq ...'
sudo apt-get install -y jq
fi
if ! [ -x "$(command -v curl)" ]; then
>&2 echo 'curl is not installed. Installing curl ...'
sudo apt-get install -y curl
fi
# Content-Type
ACCEPT_JSON="Accept: application/json"
ACCEPT_CSV="Accept: text/csv"
CONTENT_JSON="Content-Type: application/json"
CONTENT_CSV="Content-Type: text/csv"
# LOCAL
#PORT=8888
#URL=http://localhost:$PORT
# PROD
PORT=443
URL=https://lns.campusiot.imag.fr:$PORT
# Doc
URL_SWAGGER=${URL}/swagger/api.swagger.json
# Operations
# Operations
#CURL="curl --verbose"
CURL="curl -s --insecure"
#CURL="curl -s"
GET="${CURL} -X GET --header \""$ACCEPT_JSON"\""
POST="${CURL} -X POST --header \""$ACCEPT_JSON"\""
PUT="${CURL} -X PUT --header \""$ACCEPT_JSON"\""
DELETE="${CURL} -X DELETE --header \""$ACCEPT_JSON"\""
OPTIONS="${CURL} -X OPTIONS --header \""$ACCEPT_JSON"\""
HEAD="${CURL} -X HEAD --header \""$ACCEPT_JSON"\""
# ===================================
# Get OpenAPI2.0 specification of the API
# -----------------------------------
#${GET} ${URL_SWAGGER} > api.swagger.json
# ===================================
# Authenfication operations
# -----------------------------------
# Get the Bearer token for the user
rm $USERNAME.token.json
${POST} --header "$CONTENT_JSON" -d "$AUTH_JSON" ${URL}/api/internal/login > $USERNAME.token.json
TOKEN=$(jq -r '.jwt' $USERNAME.token.json)
AUTH="Grpc-Metadata-Authorization: Bearer $TOKEN"
echo "$TOKEN"
#sudo npm install -g jwt-cli
#jwt $TOKEN
#!/bin/bash
# Copyright (C) CampusIoT, - All Rights Reserved
# Written by CampusIoT Dev Team, 2016-2021
# ------------------------------------------------
# Get organizations
# ------------------------------------------------
# Parameters
if [[ $# -ne 1 ]] ; then
echo "Usage: $0 JWT"
exit 1
fi
TOKEN="$1"
AUTH="Grpc-Metadata-Authorization: Bearer $TOKEN"
#sudo npm install -g jwt-cli
#jwt $TOKEN
# Installation
if ! [ -x "$(command -v jq)" ]; then
echo 'jq is not installed. Installing jq ...'
sudo apt-get install -y jq
fi
if ! [ -x "$(command -v curl)" ]; then
echo 'curl is not installed. Installing curl ...'
sudo apt-get install -y curl
fi
# Content-Type
ACCEPT_JSON="Accept: application/json"
ACCEPT_CSV="Accept: text/csv"
CONTENT_JSON="Content-Type: application/json"
CONTENT_CSV="Content-Type: text/csv"
# LOCAL
#PORT=8888
#URL=http://localhost:$PORT
# PROD
PORT=443
URL=https://lns.campusiot.imag.fr:$PORT
# Operations
#CURL="curl --verbose"
CURL="curl --insecure"
#CURL="curl -s"
GET="${CURL} -X GET --header \""$ACCEPT_JSON"\""
POST="${CURL} -X POST --header \""$ACCEPT_JSON"\""
PUT="${CURL} -X PUT --header \""$ACCEPT_JSON"\""
DELETE="${CURL} -X DELETE --header \""$ACCEPT_JSON"\""
OPTIONS="${CURL} -X OPTIONS --header \""$ACCEPT_JSON"\""
HEAD="${CURL} -X HEAD --header \""$ACCEPT_JSON"\""
${GET} \
--header "$AUTH" ${URL}'/api/organizations?limit=1000&offset=0' \
> .organizations.json
jq '.result[] | ( .id + ": " + .name + " - " + .displayName)' .organizations.json
#!/bin/bash
# Copyright (C) CampusIoT, - All Rights Reserved
# Written by CampusIoT Dev Team, 2016-2021
# ------------------------------------------------
# Parse getaways
# ------------------------------------------------
rm .mails.csv
GWIDS=$(jq --raw-output ".result | sort_by(.lastSeenAt, .id) | reverse [] | (.id)" .gateways.json)
for GWID in $GWIDS
do
node ./parse_gateway.js .gateway-${GWID}.json >> .mails.csv
done
CONTENT_TYPE="Content-type: text/html"
while IFS="|" read -r TO SUBJECT BODY
do
# TODO add cc:
echo "$BODY" | mail -a "$CONTENT_TYPE" -s "$SUBJECT" -u monitoring $TO
done < .mails.csv
\ No newline at end of file
{
"requires": true,
"lockfileVersion": 1,
"dependencies": {
"moment": {
"version": "2.29.1",
"resolved": "https://registry.npmjs.org/moment/-/moment-2.29.1.tgz",
"integrity": "sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ=="
}
}
}
/*
# Copyright (C) CampusIoT, - All Rights Reserved
# Written by CampusIoT Dev Team, 2016-2021
*/
var argv = process.argv;
const fs = require('fs');
const moment = require('moment');
var <