module that displays a certain text from a website, the text on the website is updated daily

Amoniak

@ashishtank said in module that displays a certain text from a website, the text on the website is updated daily:

MMM-PoemOfTheDay is perfect to adapt for your need. Instead of poem site you need to get data from WatchTower site and parse the html response to extract the text.

wow, thank you very much for your feedback, I’ll try to see if I can do it.

Amoniak

@ashishtank ok, I have installed the module, but it only shows me a black screen

ashishtank

@Amoniak any error in dev console or PM2/terminal ?

Amoniak

@ashishtank i got it now, i will now try to parse the html response to extract the text, any good tips?

ashishtank

@Amoniak yes, I checked the response from watch tower in postman and they give data of last 2-3 days or may be from Sunday to this day, so you will need to select the right node. You can use class tabContent as identifier and today’s date for selection of the node. Also check which html parse is good enough, as HTML is very complex to parse you may want to choose right parser which gives speed and is lightweight. i.e. node-html-parser

Other thing you can do is once you have parsed text data from watch tower, you can assign it to poem’s format (title, content and name) and it will just work. later on you can change the UI and text property names if you want.

sdetweil

@ashishtank cheerio lib will parse html to json

Amoniak

well, I’ll get started then. Keep your fingers crossed.

Amoniak

i installed cheerio, but i don’t know how to use it.
do I have to put this now in my “MMM-PoemOfTheDay.js” ?
Bild 20.01.21 um 23.10.jpg

sdetweil

@Amoniak gotta go read the library usage

lolo

It gonna be harder than u think as your text is changing daily. Just follow the the pattern.
Every day this is changed( https://wol.jw.org/hr/wol/h/r19/lp-c).
new day new text.
Text is in :
div id=“dailyText”
but if you ignore reference in text as (Mat. 16:24, 25; Ivan 15:20) located in a class it’s gonna be more easier.
And with cheerio u can complete this.

Amoniak

so, this is what my NodeHelper.js looks like. like i thought, the screen is black.

const axios = require("axios");
const DetectLanguage = require("detectlanguage");

module.exports = NodeHelper.create({
        socketNotificationReceived: async function(noti, payload) {
                if (noti === "START") {
                        if (payload.updateInterval < 120000) {
                                payload.updateInterval = 120000;
                        }
                        const self = this;
                        (async function displayWatchtower () {
                                const poem = await getWatchtower(payload);
                                self.sendSocketNotification("UPDATE", poem);
                                setTimeout(displayPoem, payload.updateInterval);
                        })();
                }
        }
});

const cheerio = require("cheerio");
const axios = require("axios").default;

const fethHtml = async url => {
  try {
    const { data } = await axios.get(url);
    return data;
  } catch {
    console.error(`ERROR: An error occurred while trying to fetch the URL: ${url}`);
  }
};

const extracWatchtower = selector => {
  const theme = selector
    .find(".tabContent active")
    .find("[class='themeScrp'] > [class='b'])
    .text()
    .trim();

 const text = selector
    .find(".bodyTxt")
    .find("[class='section']")
    .text()
    .trim();

 return { theme,
          text
 };
};

const scrapWatchtower = async () => {
  const watchtowerUrl =
    "https://wol.jw.org/hr/wol/h/r19/lp-c";

  const html = await fethHtml(watchtowerUrl);

  const selector = cheerio.load(html);

  const searchResults = selector("body")
      .find("#dailyText"
  );

  const deals = searchResults.map((idx, el) => {
      const elementSelector = selector(el);
      return extractDeal(elementSelector)
    })
    .get();

  return Watchtower;

};

ashishtank

@Amoniak there are lots of missing pieces… add console.log with some message in each function and see how the flow works.

lolo

Try to do something with this code. It will retrieve data from current day. Modifying that node_helper is not great idea.

    const url=  'https://wol.jw.org/hr/wol/h/r19/lp-c';
    request({
        url: url,
        method: 'GET'
        }, (error, response, body) => {
            if (error) {
                return console.error(error)
            };
         
            var $ = cheerio.load(body); 
            //console.log($.html());  
            const data =  $('div[class="tabContent"]').first();
                    
            let header = $(data).find('h2').text();
            let title = $(data).find('p').first().text();
            let text = $(data).find('div.pGroup>p').text();
            //console.log(header,title,text);
            var recivedData={
                header,
                title,
                text
            };
            //console.log(recivedData);
            
      
    });

fordi

I bet you could use the same technique to see if any playstation 5 are in stock on websites?

Amoniak

@lolo thank you for the code.
I used the “MagicMirror-Module-Template” to create a new file and the associated folder.
this is my js:

‘’’

Module.register(“dnevni_citat”, {
defaults: {
updateInterval: 60000,
retryDelay: 5000
},

    requiresVersion: "2.1.0", // Required version of MagicMirror

    start: function() {
            var self = this;
            var dataRequest = null;
            var dataNotification = null;

            //Flag for check if module is loaded
            this.loaded = false;

            // Schedule update timer.
            this.getData();
            setInterval(function() {
                    self.updateDom();
            }, this.config.updateInterval);
    },

const url=  'https://wol.jw.org/hr/wol/h/r19/lp-c';
request({
    url: url,
    method: 'GET'
    }, (error, response, body) => {
        if (error) {
            return console.error(error)
        };
     
        var $ = cheerio.load(body); 
        //console.log($.html());  
        const data =  $('div[class="tabContent"]').first();
                
        let header = $(data).find('h2').text();
        let title = $(data).find('p').first().text();
        let text = $(data).find('div.pGroup>p').text();
        //console.log(header,title,text);
        var recivedData={
            header,
            title,
            text
        };
        //console.log(recivedData);
    
            }
    },

‘’’
what am I doing wrong?

lolo

This should be your node_helper.

const NodeHelper = require("node_helper");
const cheerio = require("cheerio");
const request = require('request');

module.exports = NodeHelper.create({

    start: function () {
        self = this;
        console.log("Starting node_helper for: " + this.name);
    },

    getData: function () {
        const url = 'https://wol.jw.org/hr/wol/h/r19/lp-c';
        request({
            url: url,
            method: 'GET'
        }, (error, response, body) => {
            if (error) {
                return console.error(error)
            };

            var $ = cheerio.load(body);
            //console.log($.html());
            const data = $('div[class="tabContent"]').first();

            let header = $(data).find('h2').text();
            let title = $(data).find('p').first().text();
            let text = $(data).find('div.pGroup>p').text();
            //console.log(header,title,text);
            var recivedData = {
                header,
                title,
                text
            }
			console.log(recivedData);
            
			self.sendSocketNotification('TEXT_RESULT', recivedData);
        });
    },

    socketNotificationReceived: function (notification, payload) {

        if (notification === 'GET_TEXT_DATA') {
            self.getData();
        }
    },
});

lolo

Now create MMM-Dnevni_citat.js
Inside you need first to registar module

Module.register("MMM-Dnevni_citat", {
	defaults: {
		updateInterval:  10 x  60 x 1000,       //  minute ,seconds ,milliseconds
		//retryDelay: 5000          // not needed for now
	},

Then first function will be start function

   start: function () {
        Log.info("Starting module: " + this.name);
        requiresVersion: "2.1.0";
        this.loaded = false;
        this.scheduleUpdate();

    },

with first function we are calling second ( scheduleUpdate) , how often will fetch data.

   scheduleUpdate: function () {
        setInterval(() => {
            this.getData();
        }, this.config.updateInterval);
        this.getData();
    },

Then we are calling getData function. Sending socketNotification to node_helper.

getData: function () {
        this.sendSocketNotification(''GET_TEXT_DATA'', this.config);
    },

Whe node_helper send back, next function will check if data is received and update DOM

socketNotificationReceived: function (notification, payload) {
       if (notification === "TEXT_RESULT") {
           this.textDataRecived = payload;
           this.loaded = true;
      },this.updateDom();

Now only is left to create DOM

    getDom: function () {
        var wrapper = document.createElement("div");

        if (!this.loaded) {
            wrapper.innerHTML = "LOADING";
            return wrapper;
        }
        if (this.loaded) {
       // here to add elements and styling
           return wrapper;
       }
    },

Amoniak

@lolo
i did that. it is still not displayed
i added this configuration in the config.js file:

{
module: “MMM-Dnevni_citat”,
position: “middle_center”

lolo

It’s not displayed as DOM elements are not created.
But there is another problem with cheerio, is not loading any data for Saturday and Sunday. This is because of website. Need to find another approach.

lolo

node_helper. But need to find different approach.

const NodeHelper = require("node_helper");
const cheerio = require("cheerio");
const request = require('request');
var self;
module.exports = NodeHelper.create({

    start: function () {
        self = this;
        console.log("Starting node_helper for: " + this.name);
    },

    getData: function () {
        
		const url = 'https://wol.jw.org/hr/wol/h/r19/lp-c';
        request({
            url: url,
            method: 'GET'
        }, (error, response, body) => {
            if (error) {
                return console.error("ERROR: ",error)
            }
            if (!error && response.statusCode == 200) {
            var $ = cheerio.load(body);
            //console.log($.html());
            const data = $('div[class="tabContent"]').first();//   

            let header = $(data).find('h2').text();
            let title = $(data).find('p').first().text();
            let textData = $(data).find('div.pGroup>p').text();
            //console.log(header,title,text);
            var recivedData = {
                header,
                title,
                textData
            }
			//console.log('recivedData: ',recivedData);
            
			self.sendSocketNotification('TEXT_RESULT', recivedData);
			}
        });
    },

    socketNotificationReceived: function (notification, payload) {
     var self = this;
        if (notification === 'GET_TEXT_DATA') {
            //console.log(notification,payload);
			self.getData();
        }
    },
});

MMM-Dnevni_citat.js. getDom need some styling.

Module.register("MMM-Dnevni_citat", {
    defaults: {
        updateInterval: 10 * 60 * 1000, //  minute ,seconds ,milliseconds
        //retryDelay: 5000          // not needed for now
    },
    start: function () {
        Log.info("Starting module: " + this.name);
        requiresVersion: "2.1.0";
        this.loaded = false;
        this.scheduleUpdate();

    },
    scheduleUpdate: function () {
        setInterval(() => {
            this.getData();
        }, this.config.updateInterval);
        this.getData();
    },
    getData: function () {
        console.log(' GET_TEXT_DATA ', this.config)
        this.sendSocketNotification('GET_TEXT_DATA', this.config);
    },
    socketNotificationReceived: function (notification, payload) {
        if (notification === "TEXT_RESULT") {
            this.textDataRecived = payload;
            console.log(this.textDataRecived);
            this.loaded = true;
        }
        this.updateDom();

    },

    getDom: function () {
        var wrapper = document.createElement("div");

        if (!this.loaded) {
            wrapper.innerHTML = "LOADING";
            return wrapper;
        }
        if (this.loaded) {
            var header = document.createElement("header");
            header.innerHTML = this.textDataRecived.header;
            wrapper.appendChild(header);

            var table = document.createElement('tr');
            table.classList.add("xsmall", "normal");
            var title = document.createElement('tr');
            title.innerHTML = this.textDataRecived.title;
            table.appendChild(title);
            var textData = document.createElement('tr');
            textData.innerHTML = this.textDataRecived.textData;

            table.appendChild(textData);
            table.appendChild(title);
            wrapper.appendChild(table);

        }
        return wrapper;
    },
});

config.js

		{    module: "MMM-Dnevni_citat",
             position: "middle_center"
		 },