Update the videos/playlists pull pipelines to automatically refresh t…

…he search index to prevent a race condition (#14) Co-authored-by: Cody Rodgers <[email protected]>
BrightcoveOS · Feb 3, 2023 · b545c2c · b545c2c
1 parent 20280f6
commit b545c2c
Show file tree

Hide file tree

Showing 4 changed files with 50 additions and 6 deletions.
diff --git a/Brightcove.DataExchangeFramework/Converters/WebApiEndpointConverter.cs b/Brightcove.DataExchangeFramework/Converters/WebApiEndpointConverter.cs
@@ -35,6 +35,7 @@ protected override void AddPlugins(ItemModel source, Endpoint endpoint)
                 accountSettings.AccountId = accountItem["AccountId"];
                 accountSettings.ClientId = accountItem["ClientId"];
                 accountSettings.ClientSecret = accountItem["ClientSecret"];
+                accountSettings.AccountItem = accountItem;
             }
 
             endpoint.AddPlugin(accountSettings);

diff --git a/Brightcove.DataExchangeFramework/Processors/GetPlayListsPipelineStepProcessor.cs b/Brightcove.DataExchangeFramework/Processors/GetPlayListsPipelineStepProcessor.cs
@@ -1,6 +1,8 @@
 using Brightcove.Core.Models;
 using Brightcove.Core.Services;
 using Brightcove.DataExchangeFramework.Settings;
+using Sitecore.ContentSearch;
+using Sitecore.Data.Items;
 using Sitecore.DataExchange.Attributes;
 using Sitecore.DataExchange.Contexts;
 using Sitecore.DataExchange.Models;
@@ -24,14 +26,15 @@ protected override void ProcessPipelineStep(PipelineStep pipelineStep = null, Pi
         {
             base.ProcessPipelineStep(pipelineStep, pipelineContext, logger);
 
+            //We may need to refresh the search index before syncing
+            RefreshSearchIndex(pipelineStep);
+
             service = new BrightcoveService(WebApiSettings.AccountId, WebApiSettings.ClientId, WebApiSettings.ClientSecret);
 
             var data = this.GetIterableData(WebApiSettings, pipelineStep);
             var dataSettings = new IterableDataSettings(data);
 
             pipelineContext.AddPlugin(dataSettings);
-
-            SetFolderSettings("Playlists");
         }
 
         protected virtual IEnumerable<PlayList> GetIterableData(WebApiSettings settings, PipelineStep pipelineStep)
@@ -51,5 +54,21 @@ protected virtual IEnumerable<PlayList> GetIterableData(WebApiSettings settings,
                 }
             }
         }
+
+        private void RefreshSearchIndex(PipelineStep pipelineStep)
+        {
+            Item videosFolder = Sitecore.Data.Database.GetDatabase("master").GetItem(WebApiSettings.AccountItem.Paths.FullPath + "/Videos");
+
+            Logger.Debug($"Started refreshing the search index before syncing videos... (pipeline step: {pipelineStep.Name})");
+
+            var jobs = Sitecore.ContentSearch.Maintenance.IndexCustodian.RefreshTree((SitecoreIndexableItem)videosFolder);
+
+            foreach (var job in jobs)
+            {
+                job.Wait();
+            }
+
+            Logger.Debug($"Finished refreshing the search index (pipeline step: {pipelineStep.Name})");
+        }
     }
 }
diff --git a/Brightcove.DataExchangeFramework/Processors/GetVideosPipelineStepProcessor.cs b/Brightcove.DataExchangeFramework/Processors/GetVideosPipelineStepProcessor.cs
@@ -1,6 +1,8 @@
 using Brightcove.Core.Models;
 using Brightcove.Core.Services;
 using Brightcove.DataExchangeFramework.Settings;
+using Sitecore.ContentSearch;
+using Sitecore.Data.Items;
 using Sitecore.DataExchange.Attributes;
 using Sitecore.DataExchange.Contexts;
 using Sitecore.DataExchange.Converters.PipelineSteps;
@@ -26,14 +28,15 @@ protected override void ProcessPipelineStep(PipelineStep pipelineStep = null, Pi
         {
             base.ProcessPipelineStep(pipelineStep, pipelineContext, logger);
 
+            //We may need to refresh the search index before syncing
+            RefreshSearchIndex(pipelineStep);
+
             service = new BrightcoveService(WebApiSettings.AccountId, WebApiSettings.ClientId, WebApiSettings.ClientSecret);
 
             var data = GetIterableData(pipelineStep);
             var dataSettings = new IterableDataSettings(data);
 
             pipelineContext.AddPlugin(dataSettings);
-
-            SetFolderSettings("Videos");
         }
 
         protected virtual IEnumerable<Video> GetIterableData(PipelineStep pipelineStep)
@@ -43,7 +46,7 @@ protected virtual IEnumerable<Video> GetIterableData(PipelineStep pipelineStep)
 
             for (int offset = 0; offset < totalVideosCount; offset += limit)
             {
-                foreach(Video video in service.GetVideos(offset, limit))
+                foreach (Video video in service.GetVideos(offset, limit))
                 {
                     Logger.Debug($"Found the brightcove asset {video.Id} (pipeline step: {pipelineStep.Name})");
 
@@ -53,5 +56,23 @@ protected virtual IEnumerable<Video> GetIterableData(PipelineStep pipelineStep)
                 }
             }
         }
+
+        private void RefreshSearchIndex(PipelineStep pipelineStep)
+        {            
+            Item labelsFolder = Sitecore.Data.Database.GetDatabase("master").GetItem(WebApiSettings.AccountItem.Paths.FullPath + "/Labels");
+            Item foldersFolder = Sitecore.Data.Database.GetDatabase("master").GetItem(WebApiSettings.AccountItem.Paths.FullPath + "/Folders");
+
+            Logger.Debug($"Started refreshing the search index before syncing videos... (pipeline step: {pipelineStep.Name})");
+
+            var jobs = Sitecore.ContentSearch.Maintenance.IndexCustodian.RefreshTree((SitecoreIndexableItem)labelsFolder);
+            jobs.Concat(Sitecore.ContentSearch.Maintenance.IndexCustodian.RefreshTree((SitecoreIndexableItem)foldersFolder));
+
+            foreach (var job in jobs)
+            {
+                job.Wait();
+            }
+
+            Logger.Debug($"Finished refreshing the search index (pipeline step: {pipelineStep.Name})");
+        }
     }
 }
diff --git a/Brightcove.DataExchangeFramework/Settings/WebApiSettings.cs b/Brightcove.DataExchangeFramework/Settings/WebApiSettings.cs
@@ -1,4 +1,5 @@
-using Sitecore.DataExchange;
+using Sitecore.Data.Items;
+using Sitecore.DataExchange;
 using System;
 using System.Collections.Generic;
 using System.Linq;
@@ -12,5 +13,7 @@ public class WebApiSettings : IPlugin
         public string AccountId { get; set; } = "";
         public string ClientId { get; set; } = "";
         public string ClientSecret { get; set; } = "";
+
+        public Item AccountItem { get; set; } = null;
     }
 }