Merge pull request #42 from thefeiter/create-sd-card-logging

Add a logging function to write incoming Packets to SD card

FDRS_Gateway/FDRS_Gateway.ino

@@ -17,6 +17,8 @@
 #include <ArduinoJson.h>
 #ifdef USE_WIFI
 #include <PubSubClient.h>
+#include <WiFiUdp.h>
+#include <NTPClient.h>
 #endif
 #ifdef USE_LORA
 #include <LoRa.h>
@@ -24,6 +26,10 @@
 #ifdef USE_LED
 #include <FastLED.h>
 #endif
+#ifdef USE_SD_LOG
+#include <SPI.h>
+#include <SD.h>
+#endif
 #include "fdrs_functions.h"
 
 void setup() {
@@ -51,25 +57,17 @@ void setup() {
   DBG("WiFi Connected");
   client.setServer(mqtt_server, mqtt_port);
   if (!client.connected()) {
-    DBG("Connecting MQTT...");
-    reconnect();
+    reconnect(5);
   }
-  DBG("MQTT Connected");
   client.setCallback(mqtt_callback);
 #else
   begin_espnow();
 #endif
 #ifdef USE_LORA
-  DBG("Initializing LoRa!");
-#ifdef ESP32
-  SPI.begin(SPI_SCK, SPI_MISO, SPI_MOSI);
+  begin_lora();
 #endif
-  LoRa.setPins(LORA_SS, LORA_RST, LORA_DIO0);
-  if (!LoRa.begin(FDRS_BAND)) {
-    while (1);
-  }
-  LoRa.setSpreadingFactor(FDRS_SF);
-  DBG(" LoRa initialized.");
+#ifdef USE_SD_LOG
+  begin_SD();
 #endif
   
   //DBG(sizeof(DataReading));
@@ -134,10 +132,18 @@ void loop() {
   getLoRa();
   #ifdef USE_WIFI
   if (!client.connected()) {
-    DBG("Connecting MQTT...");
-    reconnect();
+    reconnect(1, true);
+  }
+  client.loop(); // for recieving incoming messages and maintaining connection
+
+  timeClient.update();  //update internal clock if possible
+  #endif
+#ifdef USE_SD_LOG
+unsigned long current_millis = millis();
+if(current_millis-last_millis >= 1000){
+  seconds_since_reset+=(current_millis-last_millis)/1000;
+  last_millis=current_millis;
 }
-  client.loop();
 #endif
   if (newData) {
     switch (newData) {

FDRS_Gateway/README.md

@@ -28,6 +28,11 @@ LORA_BAND and LORA_SF (spreading factor) can also be configured in 'fdrs_globals
 Enables WiFi. Used only on the MQTT gateway.
 
 SSID, password, and MQTT credentials are also configurable in 'fdrs_globals.h'.
+### ```#define USE_SD_LOG```
+Enables SD-card logging. Used only on the MQTT gateway if sending the MQTT message fails. Make sure to set the correct SD_SS (chip/slave select) pin in the lines below.
+
+Logging is done in the following CSV Format: ```timestamp,reading_id,datatype,value```
+
 ### ```#define USE_LED```
 This option initializes FastLED! I haven't developed this very much, perhaps you have ideas?
 

FDRS_Gateway/fdrs_config.h

@@ -18,6 +18,7 @@
 
 //#define USE_LORA
 //#define USE_WIFI      //Used only for MQTT gateway
+//#define USE_SD_LOG      //Used only for SD-card logging
 
 // Peer addresses
 #define ESPNOW1_PEER  0x0E  // ESPNOW1 Address 
@@ -36,15 +37,16 @@
 #define TXD2 15
 
 //SPI Configuration -- Needed only on Boards with multiple SPI interfaces like the ESP32
+
 #define SPI_SCK 5
 #define SPI_MISO 19
 #define SPI_MOSI 27
 
-
 //LoRa Configuration -- Needed only if using LoRa
 #define LORA_SS 18
 #define LORA_RST 14
 #define LORA_DIO0 26
+
 //433E6 for Asia
 //866E6 for Europe
 //915E6 for North America
@@ -81,3 +83,8 @@
 #define TOPIC_DATA "fdrs/data"
 #define TOPIC_STATUS "fdrs/status"
 #define TOPIC_COMMAND "fdrs/command" 
+
+
+// SD card logging config -- Needed only for SD-card logging
+#define SD_SS 0     //SD card Chipselect pin (Use a different pins for LoRa and SD)
+#define SD_FILENAME "fdrs_log.csv"  // length max. 32

FDRS_Gateway/fdrs_functions.h

@@ -78,6 +78,11 @@ uint8_t LoRa2[] =         {mac_prefix[3], mac_prefix[4], LORA2_PEER};
 //uint8_t LoRaAddress[] = {0x42, 0x00};
 #endif
 
+#ifdef USE_SD_LOG
+unsigned long last_millis = 0;
+unsigned long seconds_since_reset = 0;
+#endif
+
 DataReading theData[256];
 uint8_t ln;
 uint8_t newData = 0;
@@ -113,6 +118,8 @@ CRGB leds[NUM_LEDS];
 #endif
 #ifdef USE_WIFI
 PubSubClient client(espClient);
+WiFiUDP ntpUDP;
+NTPClient timeClient(ntpUDP);
 const char* ssid = FDRS_WIFI_SSID;
 const char* password = FDRS_WIFI_PASS;
 const char* mqtt_server = FDRS_MQTT_ADDR;
@@ -126,6 +133,8 @@ const char* mqtt_user = NULL;
 const char* mqtt_pass = NULL;
 #endif
 
+
+
 // Set ESP-NOW send and receive callbacks for either ESP8266 or ESP32
 #if defined(ESP8266)
 void OnDataSent(uint8_t *mac_addr, uint8_t sendStatus) {
@@ -166,6 +175,56 @@ void getSerial() {
 
   }
 }
+void sendSD(const char filename[32]) {
+  #ifdef USE_SD_LOG
+  DBG("Logging to SD card.");
+  File logfile = SD.open(filename, FILE_WRITE);
+  for (int i = 0; i < ln; i++) {
+    #ifdef USE_WIFI
+    logfile.print(timeClient.getEpochTime());
+    #else
+    logfile.print(seconds_since_reset);
+    #endif
+    logfile.print(",");
+    logfile.print(theData[i].id);
+    logfile.print(",");
+    logfile.print(theData[i].t);
+    logfile.print(",");
+    logfile.println(theData[i].d);
+  }
+  logfile.close();
+  #endif
+}
+void reconnect(int attempts, bool silent) {
+#ifdef USE_WIFI
+
+  if(!silent) DBG("Connecting MQTT...");
+  
+  for (int i = 1; i<=attempts; i++) {
+    // Attempt to connect
+    if (client.connect("FDRS_GATEWAY", mqtt_user, mqtt_pass)) {
+      // Subscribe
+      client.subscribe(TOPIC_COMMAND);
+      if(!silent) DBG(" MQTT Connected");
+      return;
+    } else {
+      if(!silent) {
+        char msg[15];
+        sprintf(msg, " Attempt %d/%d",i,attempts);
+        DBG(msg);
+      }
+      if(attempts=!1){
+        delay(3000);
+      }
+    }
+  }
+
+  if(!silent) DBG(" Connecting MQTT failed.");
+#endif
+}
+void reconnect(int attempts){
+  reconnect(attempts, false);
+}
 void mqtt_callback(char* topic, byte * message, unsigned int length) {
   String incomingString;
   DBG(topic);
@@ -192,6 +251,14 @@ void mqtt_callback(char* topic, byte * message, unsigned int length) {
 
   }
 }
+void mqtt_publish(const char* payload){
+  #ifdef USE_WIFI
+  if(!client.publish(TOPIC_DATA, payload)){
+    DBG(" Error on sending MQTT");
+    sendSD(SD_FILENAME);
+  }
+  #endif
+}
 
 void getLoRa() {
 #ifdef USE_LORA
@@ -275,7 +342,7 @@ void sendMQTT() {
   }
   String outgoingString;
   serializeJson(doc, outgoingString);
-  client.publish(TOPIC_DATA, (char*) outgoingString.c_str());
+  mqtt_publish((char*) outgoingString.c_str());
 #endif
 }
 
@@ -496,25 +563,10 @@ void releaseMQTT() {
   }
   String outgoingString;
   serializeJson(doc, outgoingString);
-  client.publish(TOPIC_DATA, (char*) outgoingString.c_str());
+  mqtt_publish((char*) outgoingString.c_str());
   lenMQTT = 0;
 #endif
 }
-void reconnect() {
-#ifdef USE_WIFI
-  // Loop until reconnected
-  while (!client.connected()) {
-    // Attempt to connect
-    if (client.connect("FDRS_GATEWAY", mqtt_user, mqtt_pass)) {
-      // Subscribe
-      client.subscribe(TOPIC_COMMAND);
-    } else {
-      DBG("Connecting MQTT.");
-      delay(5000);
-    }
-  }
-#endif
-}
 void begin_espnow() {
   DBG("Initializing ESP-NOW!");
   WiFi.mode(WIFI_STA);
@@ -570,3 +622,32 @@ void begin_espnow() {
 #endif
   DBG(" ESP-NOW Initialized.");
 }
+void begin_lora(){
+  #ifdef USE_LORA
+  DBG("Initializing LoRa!");
+#ifdef ESP32
+  SPI.begin(SPI_SCK, SPI_MISO, SPI_MOSI);
+#endif
+  LoRa.setPins(LORA_SS, LORA_RST, LORA_DIO0);
+  if (!LoRa.begin(FDRS_BAND)) {
+    DBG(" Initialization failed!");
+    while (1);
+  }
+  LoRa.setSpreadingFactor(FDRS_SF);
+  DBG(" LoRa initialized.");
+  #endif
+}
+void begin_SD(){
+  #ifdef USE_SD_LOG
+  DBG("Initializing SD card...");
+  #ifdef ESP32
+  SPI.begin(SCK, MISO, MOSI);
+  #endif
+  if (!SD.begin(SD_SS)) {
+    DBG(" Initialization failed!");
+    while (1);
+  }else{
+    DBG(" SD initialized.");
+  }
+  #endif
+}

fdrs_functions.h

@@ -78,6 +78,11 @@ uint8_t LoRa2[] =         {mac_prefix[3], mac_prefix[4], LORA2_PEER};
 //uint8_t LoRaAddress[] = {0x42, 0x00};
 #endif
 
+#ifdef USE_SD_LOG
+unsigned long last_millis = 0;
+unsigned long seconds_since_reset = 0;
+#endif
+
 DataReading theData[256];
 uint8_t ln;
 uint8_t newData = 0;
@@ -113,6 +118,8 @@ CRGB leds[NUM_LEDS];
 #endif
 #ifdef USE_WIFI
 PubSubClient client(espClient);
+WiFiUDP ntpUDP;
+NTPClient timeClient(ntpUDP);
 const char* ssid = FDRS_WIFI_SSID;
 const char* password = FDRS_WIFI_PASS;
 const char* mqtt_server = FDRS_MQTT_ADDR;
@@ -126,6 +133,8 @@ const char* mqtt_user = NULL;
 const char* mqtt_pass = NULL;
 #endif
 
+
+
 // Set ESP-NOW send and receive callbacks for either ESP8266 or ESP32
 #if defined(ESP8266)
 void OnDataSent(uint8_t *mac_addr, uint8_t sendStatus) {
@@ -166,6 +175,56 @@ void getSerial() {
 
   }
 }
+void sendSD(const char filename[32]) {
+  #ifdef USE_SD_LOG
+  DBG("Logging to SD card.");
+  File logfile = SD.open(filename, FILE_WRITE);
+  for (int i = 0; i < ln; i++) {
+    #ifdef USE_WIFI
+    logfile.print(timeClient.getEpochTime());
+    #else
+    logfile.print(seconds_since_reset);
+    #endif
+    logfile.print(",");
+    logfile.print(theData[i].id);
+    logfile.print(",");
+    logfile.print(theData[i].t);
+    logfile.print(",");
+    logfile.println(theData[i].d);
+  }
+  logfile.close();
+  #endif
+}
+void reconnect(int attempts, bool silent) {
+#ifdef USE_WIFI
+
+  if(!silent) DBG("Connecting MQTT...");
+  
+  for (int i = 1; i<=attempts; i++) {
+    // Attempt to connect
+    if (client.connect("FDRS_GATEWAY", mqtt_user, mqtt_pass)) {
+      // Subscribe
+      client.subscribe(TOPIC_COMMAND);
+      if(!silent) DBG(" MQTT Connected");
+      return;
+    } else {
+      if(!silent) {
+        char msg[15];
+        sprintf(msg, " Attempt %d/%d",i,attempts);
+        DBG(msg);
+      }
+      if(attempts=!1){
+        delay(3000);
+      }
+    }
+  }
+
+  if(!silent) DBG(" Connecting MQTT failed.");
+#endif
+}
+void reconnect(int attempts){
+  reconnect(attempts, false);
+}
 void mqtt_callback(char* topic, byte * message, unsigned int length) {
   String incomingString;
   DBG(topic);
@@ -192,6 +251,14 @@ void mqtt_callback(char* topic, byte * message, unsigned int length) {
 
   }
 }
+void mqtt_publish(const char* payload){
+  #ifdef USE_WIFI
+  if(!client.publish(TOPIC_DATA, payload)){
+    DBG(" Error on sending MQTT");
+    sendSD(SD_FILENAME);
+  }
+  #endif
+}
 
 void getLoRa() {
 #ifdef USE_LORA
@@ -275,7 +342,7 @@ void sendMQTT() {
   }
   String outgoingString;
   serializeJson(doc, outgoingString);
-  client.publish(TOPIC_DATA, (char*) outgoingString.c_str());
+  mqtt_publish((char*) outgoingString.c_str());
 #endif
 }
 
@@ -496,25 +563,10 @@ void releaseMQTT() {
   }
   String outgoingString;
   serializeJson(doc, outgoingString);
-  client.publish(TOPIC_DATA, (char*) outgoingString.c_str());
+  mqtt_publish((char*) outgoingString.c_str());
   lenMQTT = 0;
 #endif
 }
-void reconnect() {
-#ifdef USE_WIFI
-  // Loop until reconnected
-  while (!client.connected()) {
-    // Attempt to connect
-    if (client.connect("FDRS_GATEWAY", mqtt_user, mqtt_pass)) {
-      // Subscribe
-      client.subscribe(TOPIC_COMMAND);
-    } else {
-      DBG("Connecting MQTT.");
-      delay(5000);
-    }
-  }
-#endif
-}
 void begin_espnow() {
   DBG("Initializing ESP-NOW!");
   WiFi.mode(WIFI_STA);
@@ -570,3 +622,32 @@ void begin_espnow() {
 #endif
   DBG(" ESP-NOW Initialized.");
 }
+void begin_lora(){
+  #ifdef USE_LORA
+  DBG("Initializing LoRa!");
+  #ifdef ESP32
+  SPI.begin(SCK, MISO, MOSI);
+  #endif
+  LoRa.setPins(SS, RST, DIO0);
+  if (!LoRa.begin(FDRS_BAND)) {
+    DBG(" Initialization failed!");
+    while (1);
+  }
+  LoRa.setSpreadingFactor(FDRS_SF);
+  DBG(" LoRa initialized.");
+  #endif
+}
+void begin_SD(){
+  #ifdef USE_SD_LOG
+  DBG("Initializing SD card...");
+  #ifdef ESP32
+  SPI.begin(SCK, MISO, MOSI);
+  #endif
+  if (!SD.begin(SD_SS)) {
+    DBG(" Initialization failed!");
+    while (1);
+  }else{
+    DBG(" SD initialized.");
+  }
+  #endif
+}