V10.1.X-4.0 (#25)

* Update the embed generator to include an optional alt-text field for iframe accessibility support * Update all Brightcove templates/items to be unpublishable to prevent unnecessary performance hits * Update the pull pipeline processors to use the ItemModelRepository and cleanup unnecessary DB requests * Update the pull pipelines to set and use a new global last sync timestamp * Update the push pipelines to use the new last sync time * Update the video push pipeline to suppor the new last sync time field * Update the value reader/writers to include consistent exception handling * Update playlist/video page size to 1000 * Update the resolve asset item pipeline to dedup content items * Add the new clean pipeline batch * Add retry logic to the brightcove service * Fix loadPlayer script * Content Updates * Fix various bugs --------- Co-authored-by: Cody Rodgers <[email protected]>
BrightcoveOS · Nov 11, 2024 · 77ab30f · 77ab30f
1 parent 62237d0
commit 77ab30f
Show file tree

Hide file tree

Showing 193 changed files with 2,966 additions and 1,758 deletions.
diff --git a/Brightcove.Core/EmbedGenerator/Models/EmbedModel.cs b/Brightcove.Core/EmbedGenerator/Models/EmbedModel.cs
@@ -30,6 +30,8 @@ public class EmbedModel
 
         public string Language { get; set; } = "";
 
+        public string Title { get; set; } = "";
+
         public EmbedModel()
         {
 

diff --git a/Brightcove.Core/Models/Experience.cs b/Brightcove.Core/Models/Experience.cs
@@ -10,9 +10,12 @@ namespace Brightcove.Core.Models
     /// </summary>
     public class Experience : Asset
     {
-        [JsonProperty("created_at", NullValueHandling = NullValueHandling.Ignore)]
+        [JsonProperty("createdAt", NullValueHandling = NullValueHandling.Ignore)]
         public DateTime CreationDate { get; set; }
 
+        [JsonProperty("updatedAt", NullValueHandling = NullValueHandling.Ignore)]
+        public new DateTime? LastModifiedDate { get; set; }
+
         [JsonProperty("publishedUrl", NullValueHandling = NullValueHandling.Ignore)]
         public string Url { get; set; }
 

diff --git a/Brightcove.Core/Models/Video.cs b/Brightcove.Core/Models/Video.cs
@@ -62,6 +62,9 @@ public class Video : Asset
         [JsonProperty("schedule", NullValueHandling = NullValueHandling.Ignore)]
         public VideoSchedule Schedule { get; set; }
 
+        [JsonProperty("variants", NullValueHandling = NullValueHandling.Ignore)]
+        public List<VideoVariant> Variants { get; set; }
+
         public Video ShallowCopy()
         {
             return (Video)this.MemberwiseClone();

diff --git a/Brightcove.Core/Services/BrightcoveHttpClient.cs b/Brightcove.Core/Services/BrightcoveHttpClient.cs
@@ -14,6 +14,7 @@ public static class BrightcoveHttpClient
         static BrightcoveHttpClient()
         {
             Instance = new HttpClient();
+            Instance.Timeout = TimeSpan.FromSeconds(30);
         }
     }
 }
diff --git a/Brightcove.Core/Services/BrightcoveService.cs b/Brightcove.Core/Services/BrightcoveService.cs
@@ -16,6 +16,9 @@ public class BrightcoveService
     {
         readonly static HttpClient client = BrightcoveHttpClient.Instance;
 
+        int retryMax = 3;
+        int retryAttempt = 0;
+
         readonly string cmsBaseUrl = "https://cms.api.brightcove.com/v1/accounts";
         readonly string ingestBaseUrl = "https://ingest.api.brightcove.com/v1/accounts";
         readonly string playersBaseUrl = "https://players.api.brightcove.com/v1/accounts";
@@ -306,6 +309,7 @@ public Video UpdateVideo(Video video)
             Video newVideo = video.ShallowCopy();
             newVideo.Id = null;
             newVideo.Images = null;
+            newVideo.Variants = null;
 
             string content = JsonConvert.SerializeObject(newVideo);
 
@@ -519,12 +523,12 @@ public PlayerList GetPlayers()
             return players;
         }
 
-        public ExperienceList GetExperiences()
+        public ExperienceList GetExperiences(string query = "", string sort = "")
         {
             HttpRequestMessage request = new HttpRequestMessage();
 
             request.Method = HttpMethod.Get;
-            request.RequestUri = new Uri($"{experienceBaseUrl}/{accountId}/experiences");
+            request.RequestUri = new Uri($"{experienceBaseUrl}/{accountId}/experiences?query={query}&sort={sort}");
 
             HttpResponseMessage response = SendRequest(request);
 
@@ -719,12 +723,12 @@ public bool TryGetPlaylist(string playlistId, out PlayList playlist)
             return true;
         }
 
-        public int VideosCount()
+        public int VideosCount(string query = "")
         {
             HttpRequestMessage request = new HttpRequestMessage();
 
             request.Method = HttpMethod.Get;
-            request.RequestUri = new Uri($"{cmsBaseUrl}/{accountId}/counts/videos");
+            request.RequestUri = new Uri($"{cmsBaseUrl}/{accountId}/counts/videos?query={query}");
 
             HttpResponseMessage response = SendRequest(request);
             Count count = JsonConvert.DeserializeObject<Count>(response.Content.ReadAsString());
@@ -747,16 +751,30 @@ public int PlayListsCount()
 
         private HttpResponseMessage SendRequest(HttpRequestMessage request)
         {
-            request.Headers.Authorization = authenticationService.CreateAuthenticationHeader();
+            try
+            {
+                request.Headers.Authorization = authenticationService.CreateAuthenticationHeader();
 
-            HttpResponseMessage response = client.Send(request);
+                HttpResponseMessage response = client.Send(request);
 
-            if (!response.IsSuccessStatusCode)
-            {
-                throw new HttpStatusException(request, response);
+                if (!response.IsSuccessStatusCode)
+                {
+                    throw new HttpStatusException(request, response);
+                }
+
+                return response;
             }
+            catch(Exception ex)
+            {
+                if(retryAttempt < retryMax)
+                {
+                    retryAttempt++;
+                    return SendRequest(request);
+                }
 
-            return response;
+                retryAttempt = 0;
+                throw ex;
+            }
         }
     }
 }
diff --git a/Brightcove.DataExchangeFramework/Brightcove.DataExchangeFramework.csproj b/Brightcove.DataExchangeFramework/Brightcove.DataExchangeFramework.csproj
@@ -438,10 +438,13 @@
     <Reference Include="System.Xml" />
   </ItemGroup>
   <ItemGroup>
+    <Compile Include="Converters\PipelineStep\ApplyMappingPipelineStepConverter.cs" />
     <Compile Include="Converters\PipelineStep\PipelineStepWithEndpointFromConverter.cs" />
     <Compile Include="Converters\NullableEnumValueReaderConverter.cs" />
+    <Compile Include="Converters\PipelineStep\SyncPipelineStepConverter.cs" />
     <Compile Include="Converters\PipelineStep\ReadAssetItemsPipelineStepConverter.cs" />
-    <Compile Include="Converters\PipelineStep\UpdatePipelineStepConverter.cs" />
+    <Compile Include="Converters\PipelineStep\UpdateAssetItemPipelineStepConverter.cs" />
+    <Compile Include="Converters\PipelineStep\UpdateVideoPipelineStepConverter.cs" />
     <Compile Include="Converters\ValueAccessor\LabelsPropertyValueAccessorConverter.cs" />
     <Compile Include="Converters\ValueAccessor\FolderPropertyValueAccessorConverter.cs" />
     <Compile Include="Converters\ValueAccessor\DateTimePropertyValueAccessorConverter.cs" />
@@ -453,6 +456,8 @@
     <Compile Include="Converters\ValueAccessor\ChainedPropertyValueAccessorConverter.cs" />
     <Compile Include="Converters\PipelineStep\ResolveOrUpdateAssetModelPipelineStepConverter.cs" />
     <Compile Include="Extensions\ItemModelExtensions.cs" />
+    <Compile Include="Helpers\BrightcoveSyncSettingsHelper.cs" />
+    <Compile Include="Helpers\IItemModelRepositoryHelper.cs" />
     <Compile Include="Helpers\ItemUpdater.cs" />
     <Compile Include="Helpers\Mapper.cs" />
     <Compile Include="Processors\BasePipelineStepProcessor.cs" />
@@ -467,18 +472,23 @@
     <Compile Include="Processors\Get\GetVideosPipelineStepProcessor.cs" />
     <Compile Include="Processors\Resolve\ResolveFolderModelPipelineStepProcessor.cs" />
     <Compile Include="Processors\Resolve\ResolveLabelModelPipelineStepProcessor.cs" />
-    <Compile Include="Processors\Resolve\ResolvePlayerModelPipelineStepProcessor.cs" />
     <Compile Include="Processors\Resolve\ResolvePlayListModelPipelineStepProcessor.cs" />
+    <Compile Include="Processors\ApplyMappingPipelineStepProcessor.cs" />
+    <Compile Include="Processors\Sync\ResetSyncPipelineStepProcessor.cs" />
+    <Compile Include="Processors\Sync\FinishSyncPipelineStepProcessor.cs" />
+    <Compile Include="Processors\Sync\VerifySyncPipelineStepProcessor.cs" />
+    <Compile Include="Processors\Sync\StartSyncPipelineStepProcessor.cs" />
+    <Compile Include="Processors\Update\UpdateAssetItemPipelineStepProcessor.cs" />
     <Compile Include="Processors\Update\UpdateLabelModelPipelineStepProcessor.cs" />
     <Compile Include="Processors\Update\UpdateFolderModelPipelineStepProcessor.cs" />
     <Compile Include="Processors\Update\UpdatePlaylistModelPipelineStepProcessor.cs" />
-    <Compile Include="Processors\Update\UpdatePlayerModelPipelineStepProcessor.cs" />
-    <Compile Include="Processors\Update\UpdateVideoItemProcessor.cs" />
+    <Compile Include="Processors\Update\UpdateVideoItemPipelineStepProcessor.cs" />
     <Compile Include="Processors\Update\UpdateVideoModelPipelineStepProcessor.cs" />
     <Compile Include="Processors\Resolve\ResolveVideoModelPipelineStepProcessor.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
     <Compile Include="SearchResults\AssetSearchResult.cs" />
     <Compile Include="Settings\EndpointsSettings.cs" />
+    <Compile Include="Settings\BrightcoveSyncSettings.cs" />
     <Compile Include="Settings\MappingSettings.cs" />
     <Compile Include="Settings\ResolveAssetItemSettings.cs" />
     <Compile Include="Settings\ResolveAssetModelSettings.cs" />

diff --git a/...htcove.DataExchangeFramework/Converters/PipelineStep/ApplyMappingPipelineStepConverter.cs b/...htcove.DataExchangeFramework/Converters/PipelineStep/ApplyMappingPipelineStepConverter.cs
@@ -0,0 +1,19 @@
+using Brightcove.DataExchangeFramework.Settings;
+using Sitecore.Data;
+using Sitecore.Data.Items;
+using Sitecore.DataExchange.ApplyMapping;
+using Sitecore.DataExchange.Attributes;
+using Sitecore.DataExchange.Models;
+using Sitecore.DataExchange.Providers.Sc.Converters.PipelineSteps;
+using Sitecore.DataExchange.Repositories;
+using Sitecore.Services.Core.Model;
+using System;
+
+namespace Brightcove.DataExchangeFramework.Converters
+{
+    [SupportedIds("{AB4AF4DF-D282-4CD1-8268-FA12A9E457A3}")]
+    public class ApplyMappingPipelineStepConverter : ApplyMappingStepConverter
+    {
+        public ApplyMappingPipelineStepConverter(IItemModelRepository repository) : base(repository) { }
+    }
+}
diff --git a/...cove.DataExchangeFramework/Converters/PipelineStep/ReadAssetItemsPipelineStepConverter.cs b/...cove.DataExchangeFramework/Converters/PipelineStep/ReadAssetItemsPipelineStepConverter.cs
@@ -31,12 +31,16 @@ protected override void AddPlugins(ItemModel source, PipelineStep pipelineStep)
 
             if (endpointId != null)
             {
-                Item endpointItem = Sitecore.Context.ContentDatabase.GetItem(new ID(endpointId));
+                ItemModel endpointItem = ItemModelRepository.Get(endpointId);
 
                 if(endpointItem != null)
                 {
-                    resolveAssetItemSettings.AcccountItemId = endpointItem["Account"];
+                    resolveAssetItemSettings.AcccountItemId = this.GetStringValue(endpointItem, "Account") ?? "";
                     resolveAssetItemSettings.RelativePath = this.GetStringValue(source, "RelativePath") ?? "";
+
+                    Database database = Sitecore.Configuration.Factory.GetDatabase(ItemModelRepository.DatabaseName);
+                    resolveAssetItemSettings.AccountItem = database.GetItem(resolveAssetItemSettings.AcccountItemId);
+                    resolveAssetItemSettings.ParentItem = database.GetItem(resolveAssetItemSettings.AccountItem?.Paths?.Path + "/" + resolveAssetItemSettings.RelativePath);
                 }
             }
 

diff --git a/...ve.DataExchangeFramework/Converters/PipelineStep/ResolveAssetItemPipelineStepConverter.cs b/...ve.DataExchangeFramework/Converters/PipelineStep/ResolveAssetItemPipelineStepConverter.cs
@@ -24,15 +24,26 @@ protected override void AddPlugins(ItemModel source, PipelineStep pipelineStep)
 
             if (endpointId != null)
             {
-                Item endpointItem = Sitecore.Context.ContentDatabase.GetItem(new ID(endpointId));
+                ItemModel endpointItem = ItemModelRepository.Get(endpointId);
 
                 if (endpointItem != null)
                 {
-                    resolveAssetItemSettings.AcccountItemId = endpointItem["Account"];
+                    resolveAssetItemSettings.AcccountItemId = this.GetStringValue(endpointItem, "Account") ?? "";
                     resolveAssetItemSettings.RelativePath = this.GetStringValue(source, "RelativePath") ?? "";
+
+                    Database database = Sitecore.Configuration.Factory.GetDatabase(ItemModelRepository.DatabaseName);
+                    resolveAssetItemSettings.AccountItem = database.GetItem(resolveAssetItemSettings.AcccountItemId);
+                    resolveAssetItemSettings.ParentItem = database.GetItem(resolveAssetItemSettings.AccountItem?.Paths?.Path + "/" + resolveAssetItemSettings.RelativePath);
                 }
             }
 
+            //We need to store the resolve asset item plugin in the global Sitecore.DataExchangeContext so it
+            //can be used in the VideoIdsPropertyValueReader
+            if (Sitecore.DataExchange.Context.GetPlugin<ResolveAssetItemSettings>() == null)
+            {
+                Sitecore.DataExchange.Context.Plugins.Add(resolveAssetItemSettings);
+            }
+
             pipelineStep.AddPlugin<ResolveAssetItemSettings>(resolveAssetItemSettings);
         }
     }

diff --git a/Brightcove.DataExchangeFramework/Converters/PipelineStep/SyncPipelineStepConverter.cs b/Brightcove.DataExchangeFramework/Converters/PipelineStep/SyncPipelineStepConverter.cs
@@ -0,0 +1,44 @@
+using Brightcove.DataExchangeFramework.Settings;
+using Sitecore.Data;
+using Sitecore.Data.Items;
+using Sitecore.DataExchange.Attributes;
+using Sitecore.DataExchange.Converters.PipelineSteps;
+using Sitecore.DataExchange.Models;
+using Sitecore.DataExchange.Providers.Sc.Converters.PipelineSteps;
+using Sitecore.DataExchange.Repositories;
+using Sitecore.Services.Core.Model;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Brightcove.DataExchangeFramework.Converters
+{
+    public class SyncPipelineStepConverter : BasePipelineStepConverter
+    {
+        public SyncPipelineStepConverter(IItemModelRepository repository) : base(repository)
+        {
+
+        }
+
+        protected override void AddPlugins(ItemModel source, PipelineStep pipelineStep)
+        {
+            Guid endpointId = this.GetGuidValue(source, "EndpointFrom");
+            BrightcoveSyncSettings settings = new BrightcoveSyncSettings();
+
+            if (endpointId != null)
+            {
+                ItemModel endpointItem = ItemModelRepository.Get(endpointId);
+
+                if(endpointItem != null)
+                {
+                    string accountId = this.GetStringValue(endpointItem, "Account") ?? "";
+                    settings.AccountItem = ItemModelRepository.Get(accountId);
+                }
+            }
+
+            pipelineStep.AddPlugin(settings);
+        }
+    }
+}
diff --git a/...ove.DataExchangeFramework/Converters/PipelineStep/UpdateAssetItemPipelineStepConverter.cs b/...ove.DataExchangeFramework/Converters/PipelineStep/UpdateAssetItemPipelineStepConverter.cs
@@ -0,0 +1,19 @@
+using Brightcove.DataExchangeFramework.Settings;
+using Sitecore.Data;
+using Sitecore.Data.Items;
+using Sitecore.DataExchange.ApplyMapping;
+using Sitecore.DataExchange.Attributes;
+using Sitecore.DataExchange.Models;
+using Sitecore.DataExchange.Providers.Sc.Converters.PipelineSteps;
+using Sitecore.DataExchange.Repositories;
+using Sitecore.Services.Core.Model;
+using System;
+
+namespace Brightcove.DataExchangeFramework.Converters
+{
+    [SupportedIds("{F4AF37B2-F92D-4E49-A017-3D7489E23910}")]
+    public class UpdateAssetItemPipelineStepConverter : UpdateSitecoreItemStepConverter
+    {
+        public UpdateAssetItemPipelineStepConverter(IItemModelRepository repository) : base(repository) { }
+    }
+}
diff --git a/...pelineStep/UpdatePipelineStepConverter.cs → ...eStep/UpdateVideoPipelineStepConverter.cs b/...pelineStep/UpdatePipelineStepConverter.cs → ...eStep/UpdateVideoPipelineStepConverter.cs
@@ -14,9 +14,9 @@
 namespace Brightcove.DataExchangeFramework.Converters
 {
     [SupportedIds("{D79A5F5C-9A5E-4B1C-B884-8E3B97CACA2D}", "{F598D123-2FE9-45D3-99E2-3E4B5063190A}")]
-    public class UpdatePipelineStepConverter : BasePipelineStepConverter
+    public class UpdateVideoPipelineStepConverter : BasePipelineStepConverter
     {
-        public UpdatePipelineStepConverter(IItemModelRepository repository) : base(repository) { }
+        public UpdateVideoPipelineStepConverter(IItemModelRepository repository) : base(repository) { }
 
         protected override void AddPlugins(ItemModel source, PipelineStep pipelineStep)
         {
@@ -59,7 +59,7 @@ protected override void AddPlugins(ItemModel source, PipelineStep pipelineStep)
                 return;
             }
 
-            Item accountItem = Sitecore.Context.ContentDatabase.GetItem(new ID(accountItemId));
+            ItemModel accountItem = ItemModelRepository.Get(accountItemId);
 
             if(accountItem == null)
             {
@@ -70,9 +70,9 @@ protected override void AddPlugins(ItemModel source, PipelineStep pipelineStep)
 
             if (accountItem != null)
             {
-                webApiSettings.AccountId = accountItem["AccountId"];
-                webApiSettings.ClientId = accountItem["ClientId"];
-                webApiSettings.ClientSecret = accountItem["ClientSecret"];
+                webApiSettings.AccountId = this.GetStringValue(accountItem, "AccountId") ?? "";
+                webApiSettings.ClientId = this.GetStringValue(accountItem, "ClientId") ?? "";
+                webApiSettings.ClientSecret = this.GetStringValue(accountItem, "ClientSecret") ?? "";
             }
 
             endpointSettings.BrightcoveEndpoint.AddPlugin(webApiSettings);

diff --git a/Brightcove.DataExchangeFramework/Converters/WebApiEndpointConverter.cs b/Brightcove.DataExchangeFramework/Converters/WebApiEndpointConverter.cs
@@ -26,16 +26,15 @@ public WebApiEndpointConverter(IItemModelRepository repository) : base(repositor
         protected override void AddPlugins(ItemModel source, Endpoint endpoint)
         {
             Guid accountItemId = this.GetGuidValue(source, TemplateAccount);
-            Item accountItem = Sitecore.Context.ContentDatabase.GetItem(new ID(accountItemId));
+            ItemModel accountItem = ItemModelRepository.Get(accountItemId);
 
             WebApiSettings accountSettings = new WebApiSettings();
 
             if(accountItem != null)
             {
-                accountSettings.AccountId = accountItem["AccountId"];
-                accountSettings.ClientId = accountItem["ClientId"];
-                accountSettings.ClientSecret = accountItem["ClientSecret"];
-                accountSettings.AccountItem = accountItem;
+                accountSettings.AccountId = this.GetStringValue(accountItem, "AccountId") ?? "";
+                accountSettings.ClientId = this.GetStringValue(accountItem, "ClientId") ?? "";
+                accountSettings.ClientSecret = this.GetStringValue(accountItem, "ClientSecret") ?? "";
             }
 
             endpoint.AddPlugin(accountSettings);