Tapping into weather station radio signals to graph weather data with Grafana and Influxdb
To make the experience fit your profile, pick a username and tell us what interests you.
We found and based on your interests.
bresser-logger_scrubbed.pyGoes on your raspberryplain - 3.11 kB - 05/24/2020 at 19:34 |
|
|
bresser-logger.serviceGoes on your raspberryservice - 226.00 bytes - 05/24/2020 at 19:31 |
|
|
JavaScript Object Notation (JSON) - 27.81 kB - 05/23/2020 at 08:44 |
|
|
JavaScript Object Notation (JSON) - 38.95 kB - 05/23/2020 at 08:44 |
|
|
JavaScript Object Notation (JSON) - 27.80 kB - 05/23/2020 at 08:44 |
|
Connecting to your TLS encrypted Influxdb can be done easily with
Leaving us with the fun task of setting up a dashboard for our data:
In case you fancy the one I did, here it is for hourly time series. I'll upload the JSON for hourly, daily and weekly in the project:
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"gnetId": null,
"graphTooltip": 1,
"id": 5,
"links": [],
"panels": [
{
"cacheTimeout": null,
"colorBackground": false,
"colorValue": false,
"colors": [
"#299c46",
"rgba(237, 129, 40, 0.89)",
"#d44a3a"
],
"datasource": null,
"decimals": 1,
"format": "celsius",
"gauge": {
"maxValue": 100,
"minValue": 0,
"show": false,
"thresholdLabels": false,
"thresholdMarkers": true
},
"gridPos": {
"h": 3,
"w": 3,
"x": 0,
"y": 0
},
"id": 12,
"interval": null,
"links": [],
"mappingType": 1,
"mappingTypes": [
{
"name": "value to text",
"value": 1
},
{
"name": "range to text",
"value": 2
}
],
"maxDataPoints": 100,
"nullPointMode": "connected",
"nullText": null,
"options": {},
"pluginVersion": "6.2.5",
"postfix": "",
"postfixFontSize": "50%",
"prefix": "",
"prefixFontSize": "50%",
"rangeMaps": [
{
"from": "null",
"text": "N/A",
"to": "null"
}
],
"sparkline": {
"fillColor": "rgba(31, 118, 189, 0.18)",
"full": false,
"lineColor": "rgb(31, 120, 193)",
"show": false
},
"tableColumn": "",
"targets": [
{
"groupBy": [
{
"params": [
"12s"
],
"type": "time"
},
{
"params": [
"null"
],
"type": "fill"
}
],
"measurement": "verstasjon",
"orderByTime": "ASC",
"policy": "48_hours",
"refId": "A",
"resultFormat": "time_series",
"select": [
[
{
"params": [
"temperature_C"
],
"type": "field"
},
{
"params": [],
"type": "mean"
}
]
],
"tags": []
}
],
"thresholds": "",
"timeFrom": null,
"timeShift": null,
"title": "Temperatur",
"transparent": true,
"type": "singlestat",
"valueFontSize": "80%",
"valueMaps": [
{
"op": "=",
"text": "N/A",
"value": "null"
}
],
"valueName": "current"
},
{
"cacheTimeout": null,
"colorBackground": false,
"colorValue": false,
"colors": [
"#299c46",
"rgba(237, 129, 40, 0.89)",
"#d44a3a"
],
"datasource": null,
"decimals": 1,
"format": "humidity",
"gauge": {
"maxValue": 100,
"minValue": 0,
"show": false,
"thresholdLabels": false,
"thresholdMarkers": true
},
"gridPos": {
"h": 3,
"w": 3,
"x": 3,
"y": 0
},
"id": 15,
"interval": null,
"links": [],
"mappingType": 1,
"mappingTypes": [
{
"name": "value to text",
"value": 1
},
{
"name": "range to text",
"value": 2
}
],
"maxDataPoints": 100,
"nullPointMode": "connected",
"nullText": null,
"options": {},
"pluginVersion": "6.2.5",
"postfix": "",
"postfixFontSize": "50%",
"prefix": "",
"prefixFontSize": "50%",
"rangeMaps": [
{
"from": "null",
"text": "N/A",
"to": "null"
}
],
"sparkline": {
"fillColor": "rgba(31, 118, 189, 0.18)",
"full": false,
"lineColor": "rgb(31, 120, 193)",
"show": false
},
"tableColumn": "",
"targets": [
{
"groupBy": [
{
"params": [
"12s"
],
"type": "time"
},
{
"params": [
"null"
],
"type": "fill"
}
],
"measurement": "verstasjon",
"orderByTime": "ASC",
"policy": "48_hours",
"refId": "A",
"resultFormat": "time_series",
"select": [
[
{
"params": [
"humidity"
],
"type": "field"
},
{
"params": [],
"type": "mean"
}
]
],
"tags": []
}
],
"thresholds": "",
"timeFrom": null,
"timeShift": null,
"title": "Luftfukt",
"transparent": true,
"type": "singlestat",
"valueFontSize": "80%",
"valueMaps": [
{
"op": "=",
"text": "N/A",
"value": "null"
}
],
"valueName": "current"
},
{
"cacheTimeout": null,
"colorBackground": false,
"colorValue": false,
"colors": [
"#299c46",
"rgba(237, 129, 40, 0.89)",
"#d44a3a"
],
"datasource": null,
"decimals": 1,
"format": "velocityms"...
Read more »
The server will need to be running Influxdb and Grafana. These have probably changed since I set them up, so install the latest releases accourding to the documentation. I highly recommend encrypting the traffic to the grafana site and the Influxdb database, which can be done using Let's Encrypt and Certbot. I'm sure there are some good tutorials around.
As for the Influxdb database, I set it up to aggregate and downsample the time series data into hourly (and 30 min), daily, weekly and monthly time series since I could find no support in Grafana for this.
cq_1h CREATE CONTINUOUS QUERY cq_1h ON verdata BEGIN SELECT mean(temperature_C) AS mean_temp, mean(humidity) AS mean_humidity, mean(wind_speed) AS mean_windspeed, max(wind_gust) AS max_windgust, mean(wind_dir_deg) AS mean_winddir, sum(rain_mm) AS sum_rain INTO verdata.autogen.verstasjon_1h FROM verdata."48_hours".verstasjon GROUP BY time(1h) END
cq_30m CREATE CONTINUOUS QUERY cq_30m ON verdata BEGIN SELECT mean(temperature_C) AS mean_temp, mean(humidity) AS mean_humidity, mean(wind_speed) AS mean_windspeed, max(wind_gust) AS max_windgust, mean(wind_dir_deg) AS mean_winddir, sum(rain_mm) AS sum_rain INTO verdata.autogen.verstasjon_30min FROM verdata."48_hours".verstasjon GROUP BY time(30m) END
cq_1d CREATE CONTINUOUS QUERY cq_1d ON verdata BEGIN SELECT mean(mean_temp) AS mean_temp, max(mean_temp) AS max_temp, min(mean_temp) AS min_temp, mean(mean_humidity) AS mean_humidity, max(mean_humidity) AS max_humidity, min(mean_humidity) AS min_humidity, mean(mean_windspeed) AS mean_windspeed, max(max_windgust) AS max_windgust, mean(mean_winddir) AS mean_winddir, sum(sum_rain) AS sum_rain INTO verdata.autogen.verstasjon_1d FROM verdata.autogen.verstasjon_30min GROUP BY time(1d) END
cq_1w CREATE CONTINUOUS QUERY cq_1w ON verdata BEGIN SELECT mean(mean_temp) AS mean_temp, max(mean_temp) AS max_temp, min(mean_temp) AS min_temp, mean(mean_humidity) AS mean_humidity, max(mean_humidity) AS max_humidity, min(mean_humidity) AS min_humidity, mean(mean_windspeed) AS mean_windspeed, max(max_windgust) AS max_windgust, mean(mean_winddir) AS mean_winddir, sum(sum_rain) AS sum_rain INTO verdata.autogen.verstasjon_1w FROM verdata.autogen.verstasjon_30min GROUP BY time(1w) END
cq_30d CREATE CONTINUOUS QUERY cq_30d ON verdata BEGIN SELECT mean(mean_temp) AS mean_temp, max(mean_temp) AS max_temp, min(mean_temp) AS min_temp, mean(mean_humidity) AS mean_humidity, max(mean_humidity) AS max_humidity, min(mean_humidity) AS min_humidity, mean(mean_windspeed) AS mean_windspeed, max(max_windgust) AS max_windgust, mean(mean_winddir) AS mean_winddir, sum(sum_rain) AS sum_rain INTO verdata.autogen.verstasjon_30d FROM verdata.autogen.verstasjon_30min GROUP BY time(30d) END
Weather data from the radio-pi will be arriwing approximately every 15th second to the influxdb database, so a retention policy was set up to discard this temporary data after 48 hours.
name duration shardGroupDuration replicaN default ---- -------- ------------------ -------- ------- autogen 0s 168h0m0s 1 false 48_hours 48h0m0s 24h0m0s 1 true
48_hours is set to the default database for incoming data, the resampled and aggregated data is stored in autogen :-)
There is already decoding software out there for a wide range of commercial weather stations which runs on the RTL SDR drivers and can be downloaded here. Using this software I built a script in python that will grab it's output, format it and upload it to my InfluxDB server. The server does not calculate the accumulated rain from the station, so we would need to do that here.
import datetime
import json
from influxdb import InfluxDBClient
from subprocess import Popen, PIPE
bresser_id = 110
prev_rain = -1 #prev_rain will be between 0 - 100.
client = InfluxDBClient(host='{host_address_goes_here}', port=8086, ssl=True, verify_ssl=True, username=''{username_goes_here}', password='{password_goes_here}')
print(client.get_list_database())
client.switch_database('{database_name_goes_here}')
# from https://gist.github.com/RobertSudwarts/acf8df23a16afdb5837f
def degrees_to_cardinal(d):
'''
note: this is highly approximate...
'''
dirs = ["N", "NNE", "NE", "ENE", "E", "ESE", "SE", "SSE",
"S", "SSW", "SW", "WSW", "W", "WNW", "NW", "NNW"]
ix = int((d + 11.25)/22.5 - 0.02)
return dirs[ix % 16]
with Popen(['rtl_433', '-f', '868.3M', '-F', 'json', '-R', '119'], stdout=PIPE, bufsize=1, universal_newlines=True) as p:
for line in p.stdout:
message = json.loads(line)
#print(message)
if message['id'] == bresser_id:
#BINGO!
utc_datetime = datetime.datetime.utcnow()
utc_datetime.strftime("%Y-%m-%d %H:%M:%S")
if prev_rain == -1:
prev_rain = message['rain_mm']
message['rain_mm'] = 0.0
elif message['rain_mm'] > prev_rain:
cur_rain = message['rain_mm'] - prev_rain
prev_rain = message['rain_mm']
message['rain_mm'] = cur_rain
elif message['rain_mm'] < prev_rain:
cur_rain = 100 - prev_rain + message['rain_mm']
prev_rain = message['rain_mm']
message['rain_mm'] = cur_rain
else:
message['rain_mm'] = 0.0
#Assumed that 0 degrees is north.
#Check direction of actual wind relative to measurements (by egesight) and compensate here
message['wind_dir_deg'] -= 180
if message['wind_dir_deg'] < 0:
message['wind_dir_deg'] += 360
winddir = degrees_to_cardinal(int(message['wind_dir_deg']))
json_body = [
{
"measurement": "verstasjon",
"tags": {
"id": message['id'],
"region": "Utstumoa",
"winddir": winddir
},
#"time": utc_datetime,
"fields": {
"wind_gust": message['wind_gust'],
"wind_dir_deg": message['wind_dir_deg'],
"temperature_C": message['temperature_C'],
"wind_speed": message['wind_speed'],
"humidity": message['humidity'],
"rain_mm": message['rain_mm']
}
}
]
if(client.write_points(json_body)):
print("uploaded to server ", utc_datetime)
else:
print("fail")
client.close()
exit() #Exit script and try new connection
else:
print("Received something, but the ID didn't match.")
Furthermore you will need to setup the script to run as a service on the radio'ed raspberry pi. Using systemd you can create a file, /etc/systemd/system/bresser-logger.service
containing something like
[Unit]
Description=BresserLogger
After=multi-user.target
[Service]
Type=simple
ExecStart=/usr/bin/python3 /home/pi/bresser-logger2.py
User=pi
WorkingDirectory=/home/pi
Restart=on-failure
[Install]
WantedBy=multi-user.target
And then
sudo systemctl start bresser-logger
The python script should now run at boot and restart if it crashes.
I swapped the RTL-SDR with the way more compact Nooelec Nano Three, and it has been working like a charm so far with my Zero:
It does too get hot, so instead of the included smol heatsinks I'll mount some larger ones as seen in the above image.
I originally planned to use this 25 USD dongle which I bought from the manufacturers official store at
https://www.rtl-sdr.com/buy-rtl-sdr-dvb-t-dongles/
And it worked fine, except an outer metal housing becoming so hot I could easily burn my fingers on it.
That thick thermal slab is hardly doing any service to the components inside...
...And it's not even connecting to the metal housing. Yikes! I'm not gonna have a radio for long if I keep this thing running 24/7.
Using some simple jQuery Ajax the Highcharts graphs now retrieves data from a Django view on my server at will. Which means events like loading the page or clicking any buttons can call my view to get fresh weather data. Thanks to Vitor for the crystal clear, cut to the chase examples on how to do this at SimpleIsBetterThanComplex. Keep in mind that I'm a total newbie on web programming, so don't take me for an authority on the subject ;-) I just piece together stuff that work using a combination of a) tutorials and examples, b) the official documentation, c) stackoverflow.com and d) assumptions from past programming experience.
Taking one of the templates from geetbootstrap.com I've mashed together this:
If only you could add 240 m/s wind gusts to your customer orders...
The basic structure around this is (and this is inspired heavily by Vitor's article):
A HTML container in my template with the special parameter "data-url":
<div id="container" data-url="{% url 'chart_data' %}">
<div id="plot1"></div>
<div id="plot2"></div>
</div>
A jQuery script in the same template which will trigger when he container element loads and query the Django site for the URL named "chart_data":
$.ajax({
dataType: 'json',
url: $("#container").attr("data-url"),
success: function (data){
console.log("Fekk tak på dataen");
renderPlot(data);
},
error: function (error){
console.log("Feil...");
console.log(error);
}
});
The "chart_data" URL will be routed through the urls.py file to the view called by the same name:
urlpatterns = [
path('', views.about),
path('data/', views.chart_data, name='chart_data'),
]
And the view will respond with a JSON object in the following matter:
def chart_data(request):
dataset = weatherHour.objects.all().order_by('timestamp').filter(timestamp__range=[datetime.now()-timedelta(days=filter_days), datetime.now()])
temperature_out_list = []
temperature_in_list = []
pressure_list = []
wind_av_list = [] #needs to be a dict for highcharts wind barbs
wind_gust_list = []
humidity_list = []
rain_list = []
for d in dataset:
temperature_out_list.append([int(d.timestamp.strftime("%s")) * 1000, d.temperature_out])
rain_list.append([int(d.timestamp.strftime("%s")) * 1000, d.rain])
temperature_in_list.append([int(d.timestamp.strftime("%s")) * 1000, d.temperature_in])
pressure_list.append([int(d.timestamp.strftime("%s")) * 1000, d.pressure])
wind_av_list.append([int(d.timestamp.strftime("%s")) * 1000, d.wind_av_ms, d.wind_deg])
wind_gust_list.append([int(d.timestamp.strftime("%s")) * 1000, d.wind_gust_ms, d.wind_deg])
humidity_list.append([int(d.timestamp.strftime("%s")) * 1000, d.humidity])
chart = {'temperature_out': temperature_out_list,
'rain': rain_list,
'temperature_in': temperature_in_list,
'pressure': pressure_list,
'wind_av': wind_av_list,
'wind_gust': wind_gust_list,
'humidity': humidity_list
}
return JsonResponse(chart, safe=False)
function renderPlot(weatherData){
console.log((weatherData));
Highcharts.chart('plot1', {
plotOptions: {
series: {
marker: {
enabled: false
}
}
},
chart: {
zoomType: 'x',
},
title: {
text: null,
},
tooltip: {
valueDecimals:1,
shared: true,
useHTML: true,
headerFormat:
'<small>{point.x:%A, %b %e, %H:%M} - {point.point.to:%H:%M}</small><br>' +
'<b>{point.point.symbolName}</b><br>'
},
xAxis: [{
type:'datetime',
crosshair: true,
},
{ // Top X axis
linkedTo: 0,
type: 'datetime',
tickInterval: 24 * 3600 * 1000,
labels: {
format: '{value:<span style="font-size: 12px; font-weight: bold">%a</span> %e. %b}',
align: 'left',
x: 3,
y: -5
},
opposite...
Read more »
It's all test data for now, generated by two python scripts and stored in my Django web app's database.
Create an account to leave a comment. Already have an account? Log In.
Become a member to follow this project and never miss any updates