show integer logs from python script as diagram in MM ?

cruunnerr · Feb 1, 2018, 9:17 PM

Hey guys,

i am trying again to get a new feature to my MM ^^

Background Story:

I finally installed a second Raspberry Pi on my heat oil tank for measuring the tank level.
Therefore i am using a HC-SR04 and a python script i found in the web.
As the script just showed the actual measurement in the console, i modified it to write the logs in a *.txt and a *.html file.
The script will be executed once a day via crontab.

The index.html will be completely overwritten every time the script will be executed. Then the file will be uploaded to my NAS.
Actually i am using an iFrame module in the MM to show the result. This was just for trying it out :)

The *.txt file is doing the longtime log of the results. At the moment it writes a time stamp and the result at the end of the file.
For future measurements i could also just print the result (in the script it is “Liter”).

Question:

My aim is to show the longtime logs in a diagram on the MM. Probably this is just possible with html and an iFrame module, right or not?
It should look like this, just in MM-style:
https://forum-raspberrypi.de/attachment/5997-monatsansicht-03-jpg/

or this: https://www.raspberrypi.org/forums/viewtopic.php?t=83808

If so, could someone help me find a good solution? Because writing html with python isn’t that easy^^
And I master neither python nor html

Here comes the tank.py:

#!/usr/bin/env python
# -*- coding: utf-8 -*-

# import required modules
import time
import datetime
import RPi.GPIO as GPIO
import os
import ftplib

GPIO.setwarnings(False)

print("Fair volume...")

# define GPIO pins
GPIOTrigger = 18
GPIOEcho    = 24

# function to measure the distance
def MeasureDistance():
  # set trigger to high
  time.sleep(0.2)
  GPIO.output(GPIOTrigger, True)

  # set trigger after 10Ã‚Âµs to low
  time.sleep(0.0001)
  GPIO.output(GPIOTrigger, False)

  # store initial start time
  StartTime = time.time()

  # store start time
  while GPIO.input(GPIOEcho) == 0:
    StartTime = time.time()

  # store stop time
  while GPIO.input(GPIOEcho) == 1:
    StopTime = time.time()

  # calculate distance
  TimeElapsed = StopTime - StartTime
  Distance = (TimeElapsed * 34400) / 2
  
  return Distance

# main function
def main():
  try:
#    while True:
      Distance0 = MeasureDistance()
      Distance01 = MeasureDistance()
      Distance02 = MeasureDistance()
      Distance03 = MeasureDistance()
      Distance04 = MeasureDistance()
      Distance05 = MeasureDistance()
      Distance06 = MeasureDistance()
      Distance07 = MeasureDistance()
      Distance08 = MeasureDistance()
      Distance09 = MeasureDistance()
      Distance10 = MeasureDistance()
      Distance11 = MeasureDistance()
      Distance12 = MeasureDistance()
      Distance13 = MeasureDistance()
      Distance14 = MeasureDistance()
      Distance15 = MeasureDistance()
      Distance16 = MeasureDistance()
      Distance17 = MeasureDistance()
      Distance18 = MeasureDistance()
      Distance19 = MeasureDistance()
      Distance20 = MeasureDistance()
      Distance_sum = Distance01 + Distance02 + Distance03 + Distance04 + Distance05 + Distance06 + Distance07 + Distance08 + Distance09 + Distance10 + Distance11 + Distance12 + Distance13 + Distance14 + Distance15 + Distance16 + Distance17 + Distance18 + Distance19 + Distance20
      Distance = round(Distance_sum / 20,1)
#    Meine Tanks haben Maximal 3.200 Liter bei 150 cm Füllhöhe
#    Zusätzlich 9 cm Offset vom Einbauort des Sensors; 
      Fuelstand = 150 - Distance
      Liter = 3200 / 141 * Fuelstand
      Zeit = time.time()
      ZeitStempel = datetime.datetime.fromtimestamp(Zeit).strftime('%Y-%m-%d_%H:%M:%S')
      print (ZeitStempel),("Entfernung: %.1f cm" % Distance),(" Fuelhoehe: %.1f cm" % Fuelstand),(" Liter: %.0f l" % Liter)
      time.sleep(1)

      Auslesezeitpunkt = datetime.datetime.fromtimestamp(Zeit).strftime('%d.%m.%Y um %H:%M:%S Uhr')

	# write result in *.txt file (add to last line)
      file = open("longtimelog.txt", "a")
      file.write(str(Auslesezeitpunkt))
      file.write(":\n")
      file.write("Ölstand = ")
      file.write(str(Liter))
      file.write("\n\n")
      file.close()

	# write result to *.html file (will be overwritten)
      file = open("index.html", "w")
      file.write("<p></p>")
      file.write(str(Auslesezeitpunkt))
      file.write(":<p>")
      file.write("Oelstand = ")
      file.write(str(Liter))
      file.write("</p>")
      file.write("<p></p><p></p><p></p>")
      file.close()

      print("Logs updated")
      time.sleep(1)
      print("Upload actual log...")

	# Lädt die index.html Datei auf das NAS
      filename = "index.html"
      ftp = ftplib.FTP("192.168.178.220")
      ftp.login("USER_NAME", "PASSWORD")
      ftp.cwd("/web")
      os.chdir(r"/home/pi")
      myfile = open("index.html", 'r')
      ftp.storlines('STOR ' + "index.html", myfile)
      myfile.close()

      time.sleep(1)

      print("Upload done")

  # reset GPIO settings if user pressed Ctrl+C
  except KeyboardInterrupt:
    print("Measurement stopped by user")
    GPIO.cleanup()

if __name__ == '__main__':
  # use GPIO pin numbering convention
  GPIO.setmode(GPIO.BCM)

  # set up GPIO pins
  GPIO.setup(GPIOTrigger, GPIO.OUT)
  GPIO.setup(GPIOEcho, GPIO.IN)

  # set trigger to false
  GPIO.output(GPIOTrigger, False)

  # call main function
  main()

Even this thought is not makable i would like to thank you guys ahead ;)

edit:

oh, and i found this here, but i don’t know if this would help.

tbbear · Feb 1, 2018, 11:40 PM

@cruunnerr Hi, i dont know if u would like what i have but i made the same u want to have.
I also use a HC-SR04 and a python script, i think its the same u use i put all results in a json file and than
i read it with my openhab server so now i can display via (MMM-Openhab) the actual filling on my mirror.
Also to get a diagram is no problem . I would realy recomment u to take a look at openhab and what it can do. In case of questions please contact me … Robert

doubleT · Feb 2, 2018, 2:07 PM

I use highcharts to make a diagram for my gas station price module. It’s quite easy to implement and use if you have the data in an object.

cruunnerr · Feb 2, 2018, 8:33 PM

@tbbear

since u wrote the brilliant Openhab module this would really be a good option :)
I will keep that in mind.

@doubleT
i will take a look at it. looks great. But i am sure i will have questions :D

edit:

i also found this: https://forum.magicmirror.builders/topic/2470/mmm-chart-view-your-graphs-on-your-mirror

But what is meant by “JSON data source”? is it just a *.js file with text in it? And is that the only file needed to use this module? i am still looking. ^^

cruunnerr · Feb 5, 2018, 4:31 AM

so i actually edited my python script, so that the results will be automatically loaded up to my MySQL Database on the Synology NAS.

With PHP i can show the Data as a diagram :)
So now i could use an iFrame module.
So far so good ^^ (never thought i get this working)

next days i will take a look at the MMM-Chart Module and the OpenHAB solution. (or a other that maybe works with MySQL)

For those, who are interested so far:

#!/usr/bin/env python
# -*- coding: utf-8 -*-

# import required modules
import time
import datetime
import RPi.GPIO as GPIO
import os
import ftplib
import mysql
import mysql.connector

GPIO.setwarnings(False)

# define server data
# ftpserver = "..."  //optional, when needed uncomment the FTP-Upload further down
# ftpuser = "..."
# ftppassword = "..."
# ftppath = "/web"

sqlhost = "..."
sqlport = "..."
sqluser = "..."
sqlpassword = "..."
sqldb = "Tank"

# define GPIO pins
GPIOTrigger = 18
GPIOEcho    = 24

# function to measure the distance
def MeasureDistance():
  # set trigger to high
  time.sleep(0.2)
  GPIO.output(GPIOTrigger, True)

  # set trigger after 10Ã‚Âµs to low
  time.sleep(0.0001)
  GPIO.output(GPIOTrigger, False)

  # store initial start time
  StartTime = time.time()

  # store start time
  while GPIO.input(GPIOEcho) == 0:
    StartTime = time.time()

  # store stop time
  while GPIO.input(GPIOEcho) == 1:
    StopTime = time.time()

  # calculate distance
  TimeElapsed = StopTime - StartTime
  Distance = (TimeElapsed * 34400) / 2
  
  return Distance

print("Messe Volumen...")

# main function
def main():
  try:
#    while True:
      Distance0 = MeasureDistance()
      Distance01 = MeasureDistance()
      Distance02 = MeasureDistance()
      Distance03 = MeasureDistance()
      Distance04 = MeasureDistance()
      Distance05 = MeasureDistance()
      Distance06 = MeasureDistance()
      Distance07 = MeasureDistance()
      Distance08 = MeasureDistance()
      Distance09 = MeasureDistance()
      Distance10 = MeasureDistance()
      Distance11 = MeasureDistance()
      Distance12 = MeasureDistance()
      Distance13 = MeasureDistance()
      Distance14 = MeasureDistance()
      Distance15 = MeasureDistance()
      Distance16 = MeasureDistance()
      Distance17 = MeasureDistance()
      Distance18 = MeasureDistance()
      Distance19 = MeasureDistance()
      Distance20 = MeasureDistance()
      Distance_sum = Distance01 + Distance02 + Distance03 + Distance04 + Distance05 + Distance06 + Distance07 + Distance08 + Distance09 + Distance10 + Distance11 + Distance12 + Distance13 + Distance14 + Distance15 + Distance16 + Distance17 + Distance18 + Distance19 + Distance20
      Distance = round(Distance_sum / 20,1)
#    Meine Tanks haben Maximal 3.200 Liter bei 150 cm Füllhöhe
#    Zusätzlich 9 cm Offset vom Einbauort des Sensors; 
      Fuelstand = 150 - Distance
      Liter = 3200 / 141 * Fuelstand
      Zeit = time.time()
      ZeitStempel = datetime.datetime.fromtimestamp(Zeit).strftime('%Y-%m-%d_%H:%M:%S')
      print (ZeitStempel),("Entfernung: %.1f cm" % Distance),(" Fuelhoehe: %.1f cm" % Fuelstand),(" Liter: %.0f l" % Liter)
      time.sleep(.1)

      Auslesezeitpunkt = datetime.datetime.fromtimestamp(Zeit).strftime('%d-%m-%Y_%H:%M:%S')
      Tag = datetime.datetime.fromtimestamp(Zeit).strftime('%Y-%m-%d')
      Uhr = datetime.datetime.fromtimestamp(Zeit).strftime('%H:%M:%S')

	# schreibe Langzeitmessung in *.csv Datei
      file = open("longtimelog.csv", "a")
      file.write(str(Tag))
      file.write(", ")
      file.write(str(Liter))
      file.write("\n")
      file.close()

        # schreibe aktuelle Messung in *.csv Datei
      file = open("log.csv", "w")
      file.write(str(Tag))
      file.write(", ")
      file.write(str(Liter))
      file.write("\n")
      file.close()

      print("Logs aktualisiert")
      time.sleep(.1)
      print("Upload Logs auf FTP...")

        # Lädt die longtimelog.csv Datei auf das NAS
#      filename = "longtimelog.csv"
#      ftp = ftplib.FTP(ftpserver)
#      ftp.login(ftpuser, ftppassword)
#      ftp.cwd(ftppath)
#      os.chdir(r"/home/pi")
#      myfile = open("longtimelog.csv", 'r')
#      ftp.storlines('STOR ' + "longtimelog.csv", myfile)
#      myfile.close()

        # Lädt die log.csv Datei auf das NAS
#      filename = "log.csv"
#      ftp = ftplib.FTP(ftpserver)
#      ftp.login(ftpuser, ftppassword)
#      ftp.cwd(ftppath)
#      os.chdir(r"/home/pi")
#      myfile = open("log.csv", 'r')
#      ftp.storlines('STOR ' + "log.csv", myfile)
#      myfile.close()

      time.sleep(.1)

      print("Verbinde mit MySQL-Datenbank...")

      time.sleep(.1)

      try:
          connection = mysql.connector.connect(host = sqlhost, port = sqlport, user = sqluser, passwd = sqlpassword, db = sqldb)
      except:
          print "Keine Verbindung zum MySQL-Server"
          exit(0)

      cursor = connection.cursor()
      cursor.execute("INSERT INTO Volumen VALUES (%s,%s)", (Tag,Liter,))
      cursor.close()
      connection.commit()

      time.sleep(.1)

      print("Upload erfolgreich")

  # reset GPIO settings if user pressed Ctrl+C
  except KeyboardInterrupt:
    print("Measurement stopped by user")
    GPIO.cleanup()

if __name__ == '__main__':
  # use GPIO pin numbering convention
  GPIO.setmode(GPIO.BCM)

  # set up GPIO pins
  GPIO.setup(GPIOTrigger, GPIO.OUT)
  GPIO.setup(GPIOEcho, GPIO.IN)

  # set trigger to false
  GPIO.output(GPIOTrigger, False)

  # call main function
  main()

ninjabreadman · Feb 5, 2018, 4:13 AM

@cruunnerr You could use a mysql module to query your database, create a route to output a JSON file, then use MMM-Charts to display.

cruunnerr · Feb 5, 2018, 4:27 AM

@ninjabreadman said in show integer logs from python script as diagram in MM ?:

create a route to output a JSON file

alright, that sound makable for me i think. ^^Will take a look. Thank you

edit: ehm…first question :D

i installed this now (seems to be newer). But where to start? Where i need to create the file, and what type of file, and how will it be loaded?
Is it a javascript file, which must be saved somewhere in the node directory?

doubleT · Feb 5, 2018, 3:09 PM

I have been following this, but I didn’t have time to comment.

You asked about the. json file. Simplyfied, it’s a file that holds a (JS) object for data sharing.

{{date:"2018-01-01", litre:500},{date:"2018-01-02", litre:498},{date:"2018-01-03", litre:495}}

Probably comparable to what you did with your txt file.
For a working solution, that would actually be enough. Grab the json, handle the object, print the result with highcharts.
To me, the SQL db seems a bit “heavy weight” for such a simple use case, but it’s solid if you can make it work. Just my opinion.

ninjabreadman · Feb 5, 2018, 4:46 PM

@cruunnerr @doubleT It would certainly be more easily handled in JSON or even a CSV (easiest to append readings). You can then set up cron to rsync the file to your mirror, load the file with an npm package like csv or fast-csv, and display in MMM-Chart (I would modify MMM-Chart to load the CSV directly).

doubleT · Feb 5, 2018, 5:11 PM

Yes, that’d be the easiest option. Could be running tonigt. ;)
Personally, I’d go with the json, though. I did it like this with Highcharts.js, which seems familiar to MMM-Charts:

node-helper.js:

const fs = require("fs");
//...
    socketNotificationReceived: function(notification, payload) {
        if (notification === "UpdateChart") {
        	this.getChart();
        }
    },
    getChart: function() {

		var rawdata = fs.readFileSync('path/to/my.json');
		var history = JSON.parse(rawdata);
		var history = history.slice(-84); // 12 per day x 7 = I only want the last 84 data points
		this.sendSocketNotification("ChartUpdate", history);
	}

module.js:

    getScripts: function() {
        return [
            this.file("highcharts/highcharts.js"),
            this.file("highcharts/series-label.js"),
            this.file("highcharts/exporting.js")
        ]
    },
    socketNotificationReceived: function(notification, payload) {
        if (notification === "ChartUpdate") {
        	this.getChart(payload);
        }
    },
    getChart: function(history) {
        var chart = "";
            for(i = 0; i < history.length; i++) {
                if (i === 0) {
	    		chart = history[i].litre;
	    	}
	    	else {
	    		chart = chart + ", " + history[i].price;
	    	}
	    }
	    var maxi = Math.max.apply(Math, JSON.parse("[" + chart + "]"));
	    var highest = maxi.toString() + " L";
	    var mini = Math.min.apply(Math, JSON.parse("[" + chart + "]"));
	    var lowest  = mini.toString() + " L";

	    Highcharts.chart('module-chart', { // the id of the div to contain the chart!
	    	chart: {
		    	height: 175, 
		    	margin: 0,
		    	left: 0
	    	},
	        plotOptions: {
	            series: {
	                marker: {
	                    enabled: false
	                }
	            }
	        },
	        tooltip: {
	            pointFormat: "Value: {point.y:.2f}"
	        },
	        yAxis: {
	            min: mini-0.05,
	            max: maxi+0.05,
	            tickInterval: 0.05,
	            plotLines: [{
	            	value: this.price,
	            	dashStyle: 'solid',
	            	width: 1,
	            	color: {
		                linearGradient: [0, 0, 900, 0],
		                stops: [
		                    [0, 'rgba(255, 255, 255, 0.1)'],
		                    [1, 'rgba(255, 255, 255, 0.2)']
		                ]
		            }
	            }, {
	            	value: Math.max.apply(Math, JSON.parse("[" + pricelist + "]")),
	            	dashStyle: 'dot',
	            	width: 1,
	            	color: {
                            linearGradient: [0, 0, 900, 0],
                            stops: [
                                [0, 'rgba(255, 255, 255, 0.2)'],
                                [1, 'rgba(255, 255, 255, 0.3)']
                            ]
                        },
                        label: {
                            text: highest,
                            x: -2,
                            y: -7
                        }
	            }, 
                    {
                    value: Math.min.apply(Math, JSON.parse("[" + pricelist + "]")),
                    dashStyle: 'dot',
                    width: 1,
                    color: {
                        linearGradient: [0, 0, 900, 0],
                        stops: [
                            [0, 'rgba(255, 255, 255, 0.2)'],
                            [1, 'rgba(255, 255, 255, 0.3)']
                        ]
                    },
                    label: {
                        text: lowest,
                        x: -2,
                        y: 13
                    }
                }]
            },
            series: [{
                data: JSON.parse("[" + pricelist + "]"), // here's the action
                step: 'center',
                lineWidth: 2,
                color: {
	            linearGradient: [0, 0, 900, 0],
	            stops: [
	                [0, 'rgba(255, 255, 255, 0.4)'],
	                [1, 'rgba(255, 255, 255, 1)']
                    ]
                }
            }]
        });
    }