• Server side software -- Grafana

    Ole Andreas Utstumo • 05/22/2020 at 19:16 • 0 comments

    Connecting to your TLS encrypted Influxdb can be done easily with

    Leaving us with the fun task of setting up a dashboard for our data:

    In case you fancy the one I did, here it is for hourly time series. I'll upload the JSON for hourly, daily and weekly in the project:

    {
  "annotations": {
    "list": [
      {
        "builtIn": 1,
        "datasource": "-- Grafana --",
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "name": "Annotations & Alerts",
        "type": "dashboard"
      }
    ]
  },
  "editable": true,
  "gnetId": null,
  "graphTooltip": 1,
  "id": 5,
  "links": [],
  "panels": [
    {
      "cacheTimeout": null,
      "colorBackground": false,
      "colorValue": false,
      "colors": [
        "#299c46",
        "rgba(237, 129, 40, 0.89)",
        "#d44a3a"
      ],
      "datasource": null,
      "decimals": 1,
      "format": "celsius",
      "gauge": {
        "maxValue": 100,
        "minValue": 0,
        "show": false,
        "thresholdLabels": false,
        "thresholdMarkers": true
      },
      "gridPos": {
        "h": 3,
        "w": 3,
        "x": 0,
        "y": 0
      },
      "id": 12,
      "interval": null,
      "links": [],
      "mappingType": 1,
      "mappingTypes": [
        {
          "name": "value to text",
          "value": 1
        },
        {
          "name": "range to text",
          "value": 2
        }
      ],
      "maxDataPoints": 100,
      "nullPointMode": "connected",
      "nullText": null,
      "options": {},
      "pluginVersion": "6.2.5",
      "postfix": "",
      "postfixFontSize": "50%",
      "prefix": "",
      "prefixFontSize": "50%",
      "rangeMaps": [
        {
          "from": "null",
          "text": "N/A",
          "to": "null"
        }
      ],
      "sparkline": {
        "fillColor": "rgba(31, 118, 189, 0.18)",
        "full": false,
        "lineColor": "rgb(31, 120, 193)",
        "show": false
      },
      "tableColumn": "",
      "targets": [
        {
          "groupBy": [
            {
              "params": [
                "12s"
              ],
              "type": "time"
            },
            {
              "params": [
                "null"
              ],
              "type": "fill"
            }
          ],
          "measurement": "verstasjon",
          "orderByTime": "ASC",
          "policy": "48_hours",
          "refId": "A",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "temperature_C"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "mean"
              }
            ]
          ],
          "tags": []
        }
      ],
      "thresholds": "",
      "timeFrom": null,
      "timeShift": null,
      "title": "Temperatur",
      "transparent": true,
      "type": "singlestat",
      "valueFontSize": "80%",
      "valueMaps": [
        {
          "op": "=",
          "text": "N/A",
          "value": "null"
        }
      ],
      "valueName": "current"
    },
    {
      "cacheTimeout": null,
      "colorBackground": false,
      "colorValue": false,
      "colors": [
        "#299c46",
        "rgba(237, 129, 40, 0.89)",
        "#d44a3a"
      ],
      "datasource": null,
      "decimals": 1,
      "format": "humidity",
      "gauge": {
        "maxValue": 100,
        "minValue": 0,
        "show": false,
        "thresholdLabels": false,
        "thresholdMarkers": true
      },
      "gridPos": {
        "h": 3,
        "w": 3,
        "x": 3,
        "y": 0
      },
      "id": 15,
      "interval": null,
      "links": [],
      "mappingType": 1,
      "mappingTypes": [
        {
          "name": "value to text",
          "value": 1
        },
        {
          "name": "range to text",
          "value": 2
        }
      ],
      "maxDataPoints": 100,
      "nullPointMode": "connected",
      "nullText": null,
      "options": {},
      "pluginVersion": "6.2.5",
      "postfix": "",
      "postfixFontSize": "50%",
      "prefix": "",
      "prefixFontSize": "50%",
      "rangeMaps": [
        {
          "from": "null",
          "text": "N/A",
          "to": "null"
        }
      ],
      "sparkline": {
        "fillColor": "rgba(31, 118, 189, 0.18)",
        "full": false,
        "lineColor": "rgb(31, 120, 193)",
        "show": false
      },
      "tableColumn": "",
      "targets": [
        {
          "groupBy": [
            {
              "params": [
                "12s"
              ],
              "type": "time"
            },
            {
              "params": [
                "null"
              ],
              "type": "fill"
            }
          ],
          "measurement": "verstasjon",
          "orderByTime": "ASC",
          "policy": "48_hours",
          "refId": "A",
          "resultFormat": "time_series",
          "select": [
            [
              {
                "params": [
                  "humidity"
                ],
                "type": "field"
              },
              {
                "params": [],
                "type": "mean"
              }
            ]
          ],
          "tags": []
        }
      ],
      "thresholds": "",
      "timeFrom": null,
      "timeShift": null,
      "title": "Luftfukt",
      "transparent": true,
      "type": "singlestat",
      "valueFontSize": "80%",
      "valueMaps": [
        {
          "op": "=",
          "text": "N/A",
          "value": "null"
        }
      ],
      "valueName": "current"
    },
    {
      "cacheTimeout": null,
      "colorBackground": false,
      "colorValue": false,
      "colors": [
        "#299c46",
        "rgba(237, 129, 40, 0.89)",
        "#d44a3a"
      ],
      "datasource": null,
      "decimals": 1,
      "format": "velocityms"...
    Read more »

  • Server side software -- Influxdb

    Ole Andreas Utstumo • 05/22/2020 at 18:43 • 0 comments

    The server will need to be running Influxdb and Grafana. These have probably changed since I set them up, so install the latest releases accourding to the documentation. I highly recommend encrypting the traffic to the grafana site and the Influxdb database, which can be done using Let's Encrypt and Certbot. I'm sure there are some good tutorials around. 

    As for the Influxdb database, I set it up to aggregate and downsample the time series data into hourly (and 30 min), daily, weekly and monthly time series since I could find no support in Grafana for this.

    cq_1h  CREATE CONTINUOUS QUERY cq_1h ON verdata BEGIN SELECT mean(temperature_C) AS mean_temp, mean(humidity) AS mean_humidity, mean(wind_speed) AS mean_windspeed, max(wind_gust) AS max_windgust, mean(wind_dir_deg) AS mean_winddir, sum(rain_mm) AS sum_rain INTO verdata.autogen.verstasjon_1h FROM verdata."48_hours".verstasjon GROUP BY time(1h) END
cq_30m CREATE CONTINUOUS QUERY cq_30m ON verdata BEGIN SELECT mean(temperature_C) AS mean_temp, mean(humidity) AS mean_humidity, mean(wind_speed) AS mean_windspeed, max(wind_gust) AS max_windgust, mean(wind_dir_deg) AS mean_winddir, sum(rain_mm) AS sum_rain INTO verdata.autogen.verstasjon_30min FROM verdata."48_hours".verstasjon GROUP BY time(30m) END
cq_1d  CREATE CONTINUOUS QUERY cq_1d ON verdata BEGIN SELECT mean(mean_temp) AS mean_temp, max(mean_temp) AS max_temp, min(mean_temp) AS min_temp, mean(mean_humidity) AS mean_humidity, max(mean_humidity) AS max_humidity, min(mean_humidity) AS min_humidity, mean(mean_windspeed) AS mean_windspeed, max(max_windgust) AS max_windgust, mean(mean_winddir) AS mean_winddir, sum(sum_rain) AS sum_rain INTO verdata.autogen.verstasjon_1d FROM verdata.autogen.verstasjon_30min GROUP BY time(1d) END
cq_1w  CREATE CONTINUOUS QUERY cq_1w ON verdata BEGIN SELECT mean(mean_temp) AS mean_temp, max(mean_temp) AS max_temp, min(mean_temp) AS min_temp, mean(mean_humidity) AS mean_humidity, max(mean_humidity) AS max_humidity, min(mean_humidity) AS min_humidity, mean(mean_windspeed) AS mean_windspeed, max(max_windgust) AS max_windgust, mean(mean_winddir) AS mean_winddir, sum(sum_rain) AS sum_rain INTO verdata.autogen.verstasjon_1w FROM verdata.autogen.verstasjon_30min GROUP BY time(1w) END
cq_30d CREATE CONTINUOUS QUERY cq_30d ON verdata BEGIN SELECT mean(mean_temp) AS mean_temp, max(mean_temp) AS max_temp, min(mean_temp) AS min_temp, mean(mean_humidity) AS mean_humidity, max(mean_humidity) AS max_humidity, min(mean_humidity) AS min_humidity, mean(mean_windspeed) AS mean_windspeed, max(max_windgust) AS max_windgust, mean(mean_winddir) AS mean_winddir, sum(sum_rain) AS sum_rain INTO verdata.autogen.verstasjon_30d FROM verdata.autogen.verstasjon_30min GROUP BY time(30d) END

    Weather data from the radio-pi will be arriwing approximately every 15th second to the influxdb database, so a retention policy was set up to discard this temporary data after 48 hours.

    name     duration shardGroupDuration replicaN default
----     -------- ------------------ -------- -------
autogen  0s       168h0m0s           1        false
48_hours 48h0m0s  24h0m0s            1        true

     48_hours is set to the default database for incoming data, the resampled and aggregated data is stored in autogen :-)

  • Client side script

    Ole Andreas Utstumo • 05/22/2020 at 17:59 • 0 comments

    There is already decoding software out there for a wide range of commercial weather stations which runs on the RTL SDR drivers and can be downloaded here. Using this software I built a script in python that will grab it's output, format it and upload it to my InfluxDB server. The server does not calculate the accumulated rain from the station, so we would need to do that here.

    import datetime
import json
from influxdb import InfluxDBClient
from subprocess import Popen, PIPE

bresser_id = 110

prev_rain = -1 #prev_rain will be between 0 - 100.

client = InfluxDBClient(host='{host_address_goes_here}', port=8086, ssl=True, verify_ssl=True, username=''{username_goes_here}', password='{password_goes_here}')
print(client.get_list_database())
client.switch_database('{database_name_goes_here}')


# from https://gist.github.com/RobertSudwarts/acf8df23a16afdb5837f
def degrees_to_cardinal(d):
    '''
    note: this is highly approximate...
    '''
    dirs = ["N", "NNE", "NE", "ENE", "E", "ESE", "SE", "SSE",
            "S", "SSW", "SW", "WSW", "W", "WNW", "NW", "NNW"]
    ix = int((d + 11.25)/22.5 - 0.02)
    return dirs[ix % 16]


with Popen(['rtl_433', '-f', '868.3M', '-F', 'json', '-R', '119'], stdout=PIPE, bufsize=1, universal_newlines=True) as p:
    for line in p.stdout:
        message = json.loads(line)
        #print(message)
        
        if message['id'] == bresser_id:
            #BINGO!
            utc_datetime = datetime.datetime.utcnow()
            utc_datetime.strftime("%Y-%m-%d %H:%M:%S")

            if prev_rain == -1:
                prev_rain = message['rain_mm']
                message['rain_mm'] = 0.0
            elif message['rain_mm'] > prev_rain:
                cur_rain = message['rain_mm'] - prev_rain
                prev_rain = message['rain_mm']
                message['rain_mm'] = cur_rain
                
            elif message['rain_mm'] < prev_rain:
                cur_rain = 100 - prev_rain + message['rain_mm']
                prev_rain = message['rain_mm']
                message['rain_mm'] = cur_rain
            else:
                message['rain_mm'] = 0.0

            #Assumed that 0 degrees is north.
            #Check direction of actual wind relative to measurements (by egesight) and compensate here
            message['wind_dir_deg'] -= 180
            if message['wind_dir_deg'] < 0:
                message['wind_dir_deg'] += 360

            winddir = degrees_to_cardinal(int(message['wind_dir_deg']))

            json_body = [
            {
                "measurement": "verstasjon",
                "tags": {
                    "id": message['id'],
                    "region": "Utstumoa",
                    "winddir": winddir
                },
                #"time": utc_datetime,
                "fields": {
                    "wind_gust": message['wind_gust'],
                    "wind_dir_deg": message['wind_dir_deg'],
                    "temperature_C": message['temperature_C'],
                    "wind_speed": message['wind_speed'],
                    "humidity": message['humidity'],
                    "rain_mm": message['rain_mm']
                }
            }
            ]
            if(client.write_points(json_body)):
                print("uploaded to server ", utc_datetime)
            else:
                print("fail")
                client.close()
                exit() #Exit script and try new connection
        else:
            print("Received something, but the ID didn't match.")

    Furthermore you will need to setup the script to run as a service on the radio'ed raspberry pi. Using systemd you can create a file, /etc/systemd/system/bresser-logger.service containing something like

    [Unit]
Description=BresserLogger
After=multi-user.target

[Service]
Type=simple
ExecStart=/usr/bin/python3 /home/pi/bresser-logger2.py
User=pi
WorkingDirectory=/home/pi
Restart=on-failure

[Install]
WantedBy=multi-user.target

     And then 

    sudo systemctl start bresser-logger

    The python script should now run at boot and restart if it crashes. 

  • Change of SDR

    Ole Andreas Utstumo • 05/22/2020 at 17:41 • 0 comments

    I swapped the RTL-SDR with the way more compact Nooelec Nano Three, and it has been working like a charm so far with my Zero:

    It does too get hot, so instead of the included smol heatsinks I'll mount some larger ones as seen in the above image. 

  • Inside the RTL-SDR

    Ole Andreas Utstumo • 05/22/2019 at 16:22 • 0 comments

    I originally planned to use this 25 USD dongle which I bought from the manufacturers official store at 

    https://www.rtl-sdr.com/buy-rtl-sdr-dvb-t-dongles/

    And it worked fine, except an outer metal housing becoming so hot I could easily burn my fingers on it.

    Undressed.

    That thick thermal slab is hardly doing any service to the components inside...

    ...And it's not even connecting to the metal housing. Yikes! I'm not gonna have a radio for long if I keep this thing running 24/7.

  • Djangoing

    Ole Andreas Utstumo • 05/14/2019 at 18:13 • 0 comments

    Using some simple jQuery Ajax the Highcharts graphs now retrieves data from a Django view on my server at will. Which means events like loading the page or clicking any buttons can call my view to get fresh weather data. Thanks to Vitor for the crystal clear, cut to the chase examples on how to do this at SimpleIsBetterThanComplex. Keep in mind that I'm a total newbie on web programming, so don't take me for an authority on the subject ;-) I just piece together stuff that work using a combination of a) tutorials and examples, b) the official documentation, c) stackoverflow.com and d) assumptions from past programming experience.

    Taking one of the templates from geetbootstrap.com I've mashed together this:

    If only you could add 240 m/s wind gusts to your customer orders...

    The basic structure around this is (and this is inspired heavily by Vitor's article):

    A HTML container in my template with the special parameter "data-url":

    <div id="container" data-url="{% url 'chart_data' %}">
    <div id="plot1"></div>
    <div id="plot2"></div>
</div>

     A jQuery script in the same template which will trigger when he container element loads and query the Django site for the URL named "chart_data":

    $.ajax({
    dataType: 'json',
    url: $("#container").attr("data-url"),
    success: function (data){
        console.log("Fekk tak på dataen");
        renderPlot(data);
    },
    error: function (error){
        console.log("Feil...");
        console.log(error);
    }
});

     The "chart_data" URL will be routed through the urls.py file to the view called by the same name:

    urlpatterns = [
    path('', views.about),
    path('data/', views.chart_data, name='chart_data'),
]

     And the view will respond with a JSON object in the following matter:

    def chart_data(request):


    dataset = weatherHour.objects.all().order_by('timestamp').filter(timestamp__range=[datetime.now()-timedelta(days=filter_days), datetime.now()])
    temperature_out_list = []
    temperature_in_list = []
    pressure_list = []
    wind_av_list = [] #needs to be a dict for highcharts wind barbs
    wind_gust_list = []
    humidity_list = []
    rain_list = []
    for d in dataset:
        temperature_out_list.append([int(d.timestamp.strftime("%s")) * 1000, d.temperature_out])
        rain_list.append([int(d.timestamp.strftime("%s")) * 1000, d.rain])
        temperature_in_list.append([int(d.timestamp.strftime("%s")) * 1000, d.temperature_in])
        pressure_list.append([int(d.timestamp.strftime("%s")) * 1000, d.pressure])
        wind_av_list.append([int(d.timestamp.strftime("%s")) * 1000, d.wind_av_ms, d.wind_deg])
        wind_gust_list.append([int(d.timestamp.strftime("%s")) * 1000, d.wind_gust_ms, d.wind_deg])
        humidity_list.append([int(d.timestamp.strftime("%s")) * 1000, d.humidity])
    chart = {'temperature_out': temperature_out_list,
        'rain': rain_list, 
        'temperature_in': temperature_in_list, 
        'pressure': pressure_list, 
        'wind_av': wind_av_list,
        'wind_gust': wind_gust_list,
        'humidity': humidity_list
        }
    return JsonResponse(chart, safe=False)
     (The weatherHour object which is part of the starting Django query is a database model where hourly weather data is stored, alongside a weatherDay model for daily aggregates for the weather. Think max, min and mean for temperature, wind, humidity, etc.)

    Finally, in our template, our data is received and our renderPlot(data) function is called:
    function renderPlot(weatherData){
    console.log((weatherData));
    
    Highcharts.chart('plot1', {
        plotOptions: {
        series: {
            marker: {
                enabled: false
            }
        }
    },
    chart: {
        zoomType: 'x',
    },
    title: {
        text: null,
    },
    tooltip: {
            valueDecimals:1,
            shared: true,
            useHTML: true,
            headerFormat:
                '<small>{point.x:%A, %b %e, %H:%M} - {point.point.to:%H:%M}</small><br>' +
                '<b>{point.point.symbolName}</b><br>'

        },
    xAxis: [{
        type:'datetime',
        crosshair: true,
        },
        { // Top X axis
        linkedTo: 0,
        type: 'datetime',
        tickInterval: 24 * 3600 * 1000,
        labels: {
            format: '{value:<span style="font-size: 12px; font-weight: bold">%a</span> %e. %b}',
            align: 'left',
            x: 3,
            y: -5
        },
        opposite...
    Read more »

  • Setting up the charts

    Ole Andreas Utstumo • 05/07/2019 at 19:31 • 0 comments

    Took me a while to "get" Highcharts, but the pieces are starting to fall in place. Luckily, there's a wealth of live examples and demos in the documentation to lend inspiration from...
    It's all test data for now, generated by two python scripts and stored in my Django web app's database.