ESP32 Flash Upgrades

* Add support for flashing bootloader and otadata partitions to ESP32 devices
* Add Project support
* Tweak models to factor out project/docs URLs

docs/source/develop/changelog.rst

@@ -6,6 +6,20 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/) because it was the first relatively standard format to pop up when I googled "changelog formats".
 
 
+[2019-02-17] - Improved ESP32 Flashing Support
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Added
+---------------------
+
+- Added support for flashing a bootloader and otadata partition to ESP32 devices
+
+
+Changed
+---------------------
+
+- SPIFFS partitions can now be flashed to ESP8266 devices
+
 
 [2019-02-15] - ThingSpeak and Grainfather Support
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

firmware_flash/admin.py

@@ -1,6 +1,6 @@
 from django.contrib import admin
 
-from firmware_flash.models import DeviceFamily, Firmware, FlashRequest
+from firmware_flash.models import DeviceFamily, Firmware, FlashRequest, Board, Project
 
 
 @admin.register(DeviceFamily)
@@ -16,3 +16,13 @@ class firmwareAdmin(admin.ModelAdmin):
 @admin.register(FlashRequest)
 class flashRequestAdmin(admin.ModelAdmin):
     list_display = ('created', 'status', 'firmware_to_flash')
+
+@admin.register(Board)
+class boardAdmin(admin.ModelAdmin):
+    list_display = ('name','family',)
+
+@admin.register(Project)
+class projectAdmin(admin.ModelAdmin):
+    list_display = ('name',)
+
+

firmware_flash/migrations/0004_update_firmware_models.py

@@ -0,0 +1,86 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.27 on 2020-02-17 22:49
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('firmware_flash', '0003_multipart'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Project',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(help_text='The name of the project the firmware is associated with', max_length=128)),
+                ('description', models.TextField(blank=True, default='', help_text='The description of the project')),
+                ('project_url', models.CharField(blank=True, default='', help_text='The URL for the project associated with the firmware', max_length=255)),
+                ('documentation_url', models.CharField(blank=True, default='', help_text='The URL for documentation/help on the firmware (if any)', max_length=255)),
+                ('support_url', models.CharField(blank=True, default='', help_text='The URL for support (if any, generally a forum thread)', max_length=255)),
+                ('weight', models.IntegerField(choices=[(1, '1 (Highest)'), (2, '2'), (3, '3'), (4, '4'), (5, '5'), (6, '6'), (7, '7'), (8, '8'), (9, '9  (Lowest)')], default=5, help_text='Weight for sorting (Lower weights rise to the top)')),
+                ('show_in_standalone_flasher', models.BooleanField(default=False, help_text='Should this show standalone flash app?')),
+            ],
+            options={
+                'verbose_name': 'Project',
+                'verbose_name_plural': 'Projects',
+            },
+        ),
+        migrations.RemoveField(
+            model_name='firmware',
+            name='documentation_url',
+        ),
+        migrations.RemoveField(
+            model_name='firmware',
+            name='project_url',
+        ),
+        migrations.AddField(
+            model_name='firmware',
+            name='checksum_bootloader',
+            field=models.CharField(blank=True, default='', help_text='SHA256 checksum of the bootloader file (for checking validity)', max_length=64),
+        ),
+        migrations.AddField(
+            model_name='firmware',
+            name='checksum_otadata',
+            field=models.CharField(blank=True, default='', help_text='SHA256 checksum of the otadata file (for checking validity)', max_length=64),
+        ),
+        migrations.AddField(
+            model_name='firmware',
+            name='download_url_bootloader',
+            field=models.CharField(blank=True, default='', help_text='The URL at which the bootloader binary can be downloaded (ESP32 only, optional)', max_length=255),
+        ),
+        migrations.AddField(
+            model_name='firmware',
+            name='download_url_otadata',
+            field=models.CharField(blank=True, default='', help_text='The URL at which the OTA Dta binary can be downloaded (ESP32 only, optional)', max_length=255),
+        ),
+        migrations.AddField(
+            model_name='firmware',
+            name='otadata_address',
+            field=models.CharField(blank=True, default='', help_text='The flash address the SPIFFS data should be flashed to (ESP32 only)', max_length=12),
+        ),
+        migrations.AlterField(
+            model_name='firmware',
+            name='download_url_spiffs',
+            field=models.CharField(blank=True, default='', help_text='The URL at which the SPIFFS binary can be downloaded (optional)', max_length=255),
+        ),
+        migrations.AlterField(
+            model_name='firmware',
+            name='revision',
+            field=models.CharField(blank=True, default='', help_text='The minor revision number', max_length=20),
+        ),
+        migrations.AlterField(
+            model_name='firmware',
+            name='spiffs_address',
+            field=models.CharField(blank=True, default='', help_text='The flash address the SPIFFS data should be flashed to', max_length=12),
+        ),
+        migrations.AddField(
+            model_name='firmware',
+            name='project',
+            field=models.ForeignKey(default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, to='firmware_flash.Project'),
+        ),
+    ]

firmware_flash/models.py

@@ -21,7 +21,7 @@
 logger = logging.getLogger(__name__)
 
 FERMENTRACK_COM_URL = "https://www.fermentrack.com"
-MODEL_VERSION = 2
+MODEL_VERSION = 3
 
 
 def check_model_version():
@@ -129,10 +129,10 @@ class Meta:
     )
 
     name = models.CharField(max_length=128, blank=False, null=False, help_text="The name of the firmware")
-    family = models.ForeignKey('DeviceFamily')
+    family = models.ForeignKey('DeviceFamily', on_delete=models.CASCADE)
 
     version = models.CharField(max_length=20, default="0.0", help_text="The major version number")
-    revision = models.CharField(max_length=20, default="0.0", help_text="The minor revision number")
+    revision = models.CharField(max_length=20, default="", help_text="The minor revision number", blank=True)
     variant = models.CharField(max_length=80, default="", blank=True,
                                help_text="The firmware 'variant' (if applicable)")
 
@@ -153,16 +153,21 @@ class Meta:
     download_url_partitions = models.CharField(max_length=255, default="", blank=True, null=False,
                                                help_text="The URL at which the partitions binary can be downloaded (ESP32 only, optional)")
     download_url_spiffs = models.CharField(max_length=255, default="", blank=True, null=False,
-                                           help_text="The URL at which the SPIFFS binary can be downloaded (ESP32 only, optional)")
+                                           help_text="The URL at which the SPIFFS binary can be downloaded (optional)")
+
+    download_url_bootloader = models.CharField(max_length=255, default="", blank=True, null=False,
+                                               help_text="The URL at which the bootloader binary can be downloaded (ESP32 only, optional)")
+
+    download_url_otadata = models.CharField(max_length=255, default="", blank=True, null=False,
+                                            help_text="The URL at which the OTA Dta binary can be downloaded (ESP32 only, optional)")
 
     spiffs_address = models.CharField(max_length=12, default="", blank=True, null=False,
-                                           help_text="The flash address the SPIFFS data is at (ESP32 only, optional)")
+                                           help_text="The flash address the SPIFFS data should be flashed to")
+
+    otadata_address = models.CharField(max_length=12, default="", blank=True, null=False,
+                                           help_text="The flash address the SPIFFS data should be flashed to (ESP32 only)")
 
 
-    project_url = models.CharField(max_length=255, default="", blank=True, null=False,
-                                   help_text="The URL for the project associated with the firmware")
-    documentation_url = models.CharField(max_length=255, default="", blank=True, null=False,
-                                         help_text="The URL for documentation/help on the firmware (if any)")
 
     weight = models.IntegerField(default=5, help_text="Weight for sorting (Lower weights rise to the top)",
                                  choices=WEIGHT_CHOICES)
@@ -173,12 +178,16 @@ class Meta:
                                            default="", blank=True)
     checksum_spiffs = models.CharField(max_length=64, help_text="SHA256 checksum of the SPIFFS file (for checking validity)",
                                        default="", blank=True)
+    checksum_bootloader = models.CharField(max_length=64, help_text="SHA256 checksum of the bootloader file (for checking validity)",
+                                           default="", blank=True)
+    checksum_otadata = models.CharField(max_length=64, help_text="SHA256 checksum of the otadata file (for checking validity)",
+                                       default="", blank=True)
+
+
+    project = models.ForeignKey('Project', on_delete=models.SET_NULL, default=None, null=True)
 
     def __str__(self):
-        name = self.name + " - " + self.version + " - " + self.revision
-        if len(self.variant) > 0:
-            name += " - " + self.variant
-        return name
+        return self.name + " - " + self.version + " - " + self.revision + " - " + self.variant
 
     def __unicode__(self):
         return self.__str__()
@@ -197,23 +206,21 @@ def load_from_website():
             Firmware.objects.all().delete()
             # Then loop through the data we received and recreate it again
             for row in data:
-                try:
-                    # This gets wrapped in a try/except as I don't want this failing if the local copy of Fermentrack
-                    # is slightly behind what is available at Fermentrack.com (eg - if there are new device families)
-                    newFirmware = Firmware(
-                        name=row['name'], version=row['version'], revision=row['revision'], family_id=row['family_id'],
-                        variant=row['variant'], is_fermentrack_supported=row['is_fermentrack_supported'],
-                        in_error=row['in_error'], description=row['description'],
-                        variant_description=row['variant_description'], download_url=row['download_url'],
-                        project_url=row['project_url'], documentation_url=row['documentation_url'], weight=row['weight'],
-                        download_url_partitions=row['download_url_partitions'],
-                        download_url_spiffs=row['download_url_spiffs'], checksum=row['checksum'],
-                        checksum_partitions=row['checksum_partitions'], checksum_spiffs=row['checksum_spiffs'],
-                        spiffs_address=row['spiffs_address'],
-                    )
-                    newFirmware.save()
-                except:
-                    pass
+                newFirmware = Firmware(
+                    name=row['name'], version=row['version'], revision=row['revision'], family_id=row['family_id'],
+                    variant=row['variant'], is_fermentrack_supported=row['is_fermentrack_supported'],
+                    in_error=row['in_error'], description=row['description'],
+                    variant_description=row['variant_description'], download_url=row['download_url'],weight=row['weight'],
+                    download_url_partitions=row['download_url_partitions'],
+                    download_url_spiffs=row['download_url_spiffs'], checksum=row['checksum'],
+                    checksum_partitions=row['checksum_partitions'], checksum_spiffs=row['checksum_spiffs'],
+                    spiffs_address=row['spiffs_address'], project_id=row['project_id'],
+                    download_url_bootloader=row['download_url_bootloader'],
+                    checksum_bootloader=row['checksum_bootloader'],
+                    download_url_otadata=row['download_url_otadata'],
+                    otadata_address=row['otadata_address'], checksum_otadata=row['checksum_otadata'],
+                )
+                newFirmware.save()
 
             return True  # Firmware table is updated
         return False  # We didn't get data back from Fermentrack.com, or there was an error
@@ -286,6 +293,16 @@ def download_to_file(self, check_checksum=True, force_download=False):
                                       self.checksum_spiffs, check_checksum, force_download):
                 return False
 
+        if len(self.download_url_bootloader) > 12:
+            if not self.download_file(self.full_filepath("bootloader"), self.download_url_bootloader,
+                                      self.checksum_bootloader, check_checksum, force_download):
+                return False
+
+        if len(self.download_url_otadata) > 12 and len(self.otadata_address) > 2:
+            if not self.download_file(self.full_filepath("otadata"), self.download_url_otadata,
+                                      self.checksum_otadata, check_checksum, force_download):
+                return False
+
         # Always download the main firmware
         return self.download_file(self.full_filepath("firmware"), self.download_url, self.checksum, check_checksum, force_download)
 
@@ -309,7 +326,7 @@ class Meta:
 
     name = models.CharField(max_length=128, blank=False, null=False, help_text="The name of the board")
 
-    family = models.ForeignKey('DeviceFamily')
+    family = models.ForeignKey('DeviceFamily', on_delete=models.CASCADE)
 
     description = models.TextField(default="", blank=True, null=False, help_text="The description of the board")
 
@@ -322,9 +339,6 @@ class Meta:
     def __str__(self):
         return self.name + " - " + str(self.family)
 
-    def __unicode__(self):
-        return self.name + " - " + unicode(self.family)
-
     @staticmethod
     def load_from_website():
         try:
@@ -392,3 +406,62 @@ def succeed(self, result_text, flash_output=""):
         self.status = self.STATUS_FINISHED
         self.save()
         return True
+
+
+class Project(models.Model):
+    class Meta:
+        verbose_name = "Project"
+        verbose_name_plural = "Projects"
+
+    WEIGHT_CHOICES = (
+        (1, "1 (Highest)"),
+        (2, "2"),
+        (3, "3"),
+        (4, "4"),
+        (5, "5"),
+        (6, "6"),
+        (7, "7"),
+        (8, "8"),
+        (9, "9  (Lowest)"),
+    )
+
+    name = models.CharField(max_length=128, blank=False, null=False,
+                            help_text="The name of the project the firmware is associated with")
+    description = models.TextField(default="", blank=True, null=False, help_text="The description of the project")
+    project_url = models.CharField(max_length=255, default="", blank=True, null=False,
+                                   help_text="The URL for the project associated with the firmware")
+    documentation_url = models.CharField(max_length=255, default="", blank=True, null=False,
+                                         help_text="The URL for documentation/help on the firmware (if any)")
+    support_url = models.CharField(max_length=255, default="", blank=True, null=False,
+                                         help_text="The URL for support (if any, generally a forum thread)")
+    weight = models.IntegerField(default=5, help_text="Weight for sorting (Lower weights rise to the top)",
+                                 choices=WEIGHT_CHOICES)
+    show_in_standalone_flasher = models.BooleanField(default=False, help_text="Should this show standalone flash app?")
+
+    def __str__(self):
+        return self.name
+
+    @staticmethod
+    def load_from_website():
+        try:
+            url = FERMENTRACK_COM_URL + "/api/project_list/all/"
+            response = requests.get(url)
+            data = response.json()
+        except:
+            return False
+
+        if len(data) > 0:
+            # If we got data, clear out the cache of Firmware
+            Project.objects.all().delete()
+            # Then loop through the data we received and recreate it again
+            for row in data:
+                newProject = Project(
+                    name=row['name'], project_url=row['project_url'], documentation_url=row['documentation_url'], weight=row['weight'],
+                    support_url=row['support_url'], id=row['id'], description=row['description']
+                )
+                newProject.save()
+
+            return True  # Project table is updated
+        return False  # We didn't get data back from Fermentrack.com, or there was an error
+
+

firmware_flash/tasks.py

@@ -40,21 +40,40 @@ def flash_firmware(flash_request_id):
         flash_cmd.append(str(arg).replace("{serial_port}", flash_request.serial_port).replace("{firmware_path}",
                                                                                               firmware_path))
 
-    # For ESP32 devices only, we need to also check to see if we need to flash partitions or SPIFFS
+    # For ESP32 devices only, we need to check if we want to flash partitions or a bootloader. I may need to add
+    # ESP8266 support for flashing a bootloader later - if I do, the code for adding the bootloader command to the
+    # below needs to be moved to the SPIFFS/
     if flash_request.board_type.family.detection_family == models.DeviceFamily.DETECT_ESP32:
         # First, check if we have a partitions file to flash
         if len(flash_request.firmware_to_flash.download_url_partitions) > 0 and len(flash_request.firmware_to_flash.checksum_partitions) > 0:
             # We need to flash partitions. Partitions are (currently) always at 0x8000
             flash_cmd.append("0x8000")
             flash_cmd.append(flash_request.firmware_to_flash.full_filepath("partitions"))
 
-        # Then, check for SPIFFS
+        if len(flash_request.firmware_to_flash.download_url_bootloader) > 0 and \
+                 len(flash_request.firmware_to_flash.checksum_bootloader) > 0:
+            # The ESP32 bootloader is always flashed to 0x1000
+            flash_cmd.append("0x1000")
+            flash_cmd.append(flash_request.firmware_to_flash.full_filepath("bootloader"))
+
+
+
+    # SPIFFS (and maybe otadata?) flashing can be done on either the ESP8266 or the ESP32
+    if flash_request.firmware_to_flash.family.flash_method == models.DeviceFamily.FLASH_ESP:
+        # Check for SPIFFS first
         if len(flash_request.firmware_to_flash.download_url_spiffs) > 0 and \
                  len(flash_request.firmware_to_flash.checksum_spiffs) > 0 and \
                  len(flash_request.firmware_to_flash.spiffs_address) > 2:
             # We need to flash SPIFFS. The location is dependent on the partition scheme, so we need to use the address
             flash_cmd.append(flash_request.firmware_to_flash.spiffs_address)
             flash_cmd.append(flash_request.firmware_to_flash.full_filepath("spiffs"))
+        # Then check for otadata
+        if len(flash_request.firmware_to_flash.download_url_otadata) > 0 and \
+                 len(flash_request.firmware_to_flash.checksum_otadata) > 0 and \
+                 len(flash_request.firmware_to_flash.otadata_address) > 2:
+            # We need to flash otadata. The location is dependent on the partition scheme, so we need to use the address
+            flash_cmd.append(flash_request.firmware_to_flash.otadata_address)
+            flash_cmd.append(flash_request.firmware_to_flash.full_filepath("otadata"))
 
 
     # TODO - Explicitly need to disable any device on that port