From 630bd8d833b9a0eb1ea54e610ca7f1e3e2251179 Mon Sep 17 00:00:00 2001 From: Lilian Delouvy <58565718+lilian-delouvy@users.noreply.github.com> Date: Fri, 22 Dec 2023 12:04:02 +0100 Subject: [PATCH] fix(ApiCall): prevent ML-CLI from generating null results for api call (#299) --- .../src/Local/FileTreatment/FileTreatment.js | 168 +++++++++--------- .../Local/FileTreatment/FileTreatment.spec.js | 105 +++++++++++ src/MlCli/JobApiCall/Job.cs | 36 +++- 3 files changed, 219 insertions(+), 90 deletions(-) create mode 100644 src/Ecotag/ClientApp/src/Local/FileTreatment/FileTreatment.spec.js diff --git a/src/Ecotag/ClientApp/src/Local/FileTreatment/FileTreatment.js b/src/Ecotag/ClientApp/src/Local/FileTreatment/FileTreatment.js index 54f4f7a7..7b25a10e 100644 --- a/src/Ecotag/ClientApp/src/Local/FileTreatment/FileTreatment.js +++ b/src/Ecotag/ClientApp/src/Local/FileTreatment/FileTreatment.js @@ -33,8 +33,80 @@ const timeSideVariables = [ {value: 'Right', label: 'Right'} ]; -const FileTreatment = ({state, setState, MonacoEditor, fetch}) => { +const checkboxOptions = [ + { + key: "checkbox_isAnnotationOpen", + id: "is_annotation_open_checkbox", + value: "test", + label: "Is annotation open ? ", + disabled: false + } +]; + +export const mapItems = data => { + const mappedItems = data.map(item => { + if (!item || !item.Left || !item.Right) { + if(item){ + console.log("File " + item.FileName + " does not contain the required data, and will be ignored. Please investigate on this file."); + console.log(item); + } + return; + } + return { + fileName: item.FileName, + left: { + Url: item.Left.Url, + FileName: item.Left.FileName, + FileDirectory: item.Left.FileDirectory, + ImageDirectory: item.Left.ImageDirectory, + FrontDefaultStringsMatcher: item.Left.FrontDefaultStringsMatcher, + StatusCode: item.Left.StatusCode, + Body: item.Left.Body, + Headers: item.Left.Headers, + TimeMs: item.Left.TimeMs, + TicksAt: item.Left.TicksAt + }, + right: { + Url: item.Right.Url, + FileName: item.Right.FileName, + FileDirectory: item.Right.FileDirectory, + ImageDirectory: item.Right.ImageDirectory, + FrontDefaultStringsMatcher: item.Right.FrontDefaultStringsMatcher, + StatusCode: item.Right.StatusCode, + Body: item.Right.Body, + Headers: item.Right.Headers, + TimeMs: item.Right.TimeMs, + TicksAt: item.Right.TicksAt + }, + id: cuid(), + parse: false, + collapse: true, + }; + }); + return mappedItems.filter(item => item !== undefined && item !== null); +}; +export const compareStatusCode = (status, result) => { + const strStatus = status.toString(); + const index = result.findIndex(element => element.value === strStatus); + if (index !== -1) { + return; + } + const newItem = {value: strStatus, label: strStatus}; + result.push(newItem); +}; + +export const setStatusFilterItems = newItems => { + const allCodes = {value: "All", label: "All"}; + const result = [allCodes]; + newItems.forEach(item => { + compareStatusCode(item.left.StatusCode, result); + compareStatusCode(item.right.StatusCode, result); + }); + return result; +}; + +const FileTreatment = ({state, setState, MonacoEditor, fetch}) => { const [filterState, setFilterState] = useState({ filterName: "KO", extensionName: "All", @@ -71,90 +143,22 @@ try { }` }); - const checkboxOptions = [ - { - key: "checkbox_isAnnotationOpen", - id: "is_annotation_open_checkbox", - value: "test", - label: "Is annotation open ? ", - disabled: false - } - ]; - - const mapItems = data => (data.map(item => { - if (item.Left == null || item.Right == null) { - return; - } - return { - fileName: item.FileName, - left: { - Url: item.Left.Url, - FileName: item.Left.FileName, - FileDirectory: item.Left.FileDirectory, - ImageDirectory: item.Left.ImageDirectory, - FrontDefaultStringsMatcher: item.Left.FrontDefaultStringsMatcher, - StatusCode: item.Left.StatusCode, - Body: item.Left.Body, - Headers: item.Left.Headers, - TimeMs: item.Left.TimeMs, - TicksAt: item.Left.TicksAt - }, - right: { - Url: item.Right.Url, - FileName: item.Right.FileName, - FileDirectory: item.Right.FileDirectory, - ImageDirectory: item.Right.ImageDirectory, - FrontDefaultStringsMatcher: item.Right.FrontDefaultStringsMatcher, - StatusCode: item.Right.StatusCode, - Body: item.Right.Body, - Headers: item.Right.Headers, - TimeMs: item.Right.TimeMs, - TicksAt: item.Right.TicksAt - }, - id: cuid(), - parse: false, - collapse: true, - }; - })); - - const compareStatusCode = (status, result) => { - const strStatus = status.toString(); - const index = result.findIndex(element => element.value === strStatus); - if (index !== -1) { - return; - } - const newItem = {value: strStatus, label: strStatus}; - result.push(newItem); - }; - - const setStatusFilterItems = newItems => { - const allCodes = {value: "All", label: "All"}; - const result = [allCodes]; - newItems.forEach(item => { - compareStatusCode(item.left.StatusCode, result); - compareStatusCode(item.right.StatusCode, result); - }); - return result; - }; - const onLoad = (result, fileName) => { - const isVersion0 = Array.isArray(result); - if (!result.hasOwnProperty('CompareLocation')) { + if(!result || !Array.isArray(result.Content) || !result.CompareLocation){ onLoadFailure(fileName); - } else { - const location = isVersion0 ? "" : result.CompareLocation; - const mappedItems = mapItems(isVersion0 ? result : result.Content); - const statusCodeItems = setStatusFilterItems(mappedItems); - setState - ({ - ...state, - fileName: fileName, - compareLocation: location, - items: mappedItems, - statusCodes: statusCodeItems - }); - setFilterState({...filterState, loadFileError: false}); } + const location = result.CompareLocation; + const mappedItems = mapItems(result.Content); + const statusCodeItems = setStatusFilterItems(mappedItems); + setState + ({ + ...state, + fileName: fileName, + compareLocation: location, + items: mappedItems, + statusCodes: statusCodeItems + }); + setFilterState({...filterState, loadFileError: false}); }; const onLoadFailure = fileName => { diff --git a/src/Ecotag/ClientApp/src/Local/FileTreatment/FileTreatment.spec.js b/src/Ecotag/ClientApp/src/Local/FileTreatment/FileTreatment.spec.js new file mode 100644 index 00000000..8e6872f0 --- /dev/null +++ b/src/Ecotag/ClientApp/src/Local/FileTreatment/FileTreatment.spec.js @@ -0,0 +1,105 @@ +import {compareStatusCode, mapItems, setStatusFilterItems,} from './FileTreatment'; + +describe('Check file treatment', () => { + it('should map items properly', () => { + const data = [ + { + FileName: "MyFirstFileName.txt", + Left: { + Url: "http://localhost:5000", + FileName: "MyFirstFileName.txt", + FileDirectory: "folder1", + ImageDirectory: "folder2", + FrontDefaultStringsMatcher: "", + StatusCode: 200, + Body: "Blah blah blah", + Headers: "", + TimeMs: 4000, + TicksAt: 0 + }, + Right: { + Url: "http://localhost:5000", + FileName: "MyFirstFileName.txt", + FileDirectory: "folder1", + ImageDirectory: "folder2", + FrontDefaultStringsMatcher: "", + StatusCode: 400, + Body: "Blah blah blah", + Headers: "", + TimeMs: 4000, + TicksAt: 0 + } + }, + undefined, + { + FileName: "MySecondFileName.txt", + Left: { + Url: "http://localhost:5000", + FileName: "MySecondFileName.txt", + FileDirectory: "folder1", + ImageDirectory: "folder2", + FrontDefaultStringsMatcher: "", + StatusCode: 200, + Body: "Blah blah blah", + Headers: "", + TimeMs: 4000, + TicksAt: 0 + }, + Right: { + Url: "http://localhost:5000", + FileName: "MySecondFileName.txt", + FileDirectory: "folder1", + ImageDirectory: "folder2", + FrontDefaultStringsMatcher: "", + StatusCode: 400, + Body: "Blah blah blah", + Headers: "", + TimeMs: 4000, + TicksAt: 0 + } + }, + null + ]; + const mappedData = mapItems(data); + expect(mappedData.length).toEqual(2); + for(let item in mappedData){ + expect(item).not.toBeUndefined(); + expect(item).not.toBeNull(); + } + }); + it('should add new status code properly', () => { + const statusCode = 200; + const result = [{value: "All", label: "All"}]; + compareStatusCode(statusCode, result); + + expect(result.length).toEqual(2); + expect(result[0]).toEqual({value: "All", label: "All"}); + expect(result[1]).toEqual({value: "200", label: "200"}); + }); + it('should set status filter items properly', () => { + const items = [ + { + left: { + StatusCode: 200 + }, + right: { + StatusCode: 200 + } + }, + { + left: { + StatusCode: 200 + }, + right: { + StatusCode: 400 + } + } + ]; + const newFilters = setStatusFilterItems(items); + + expect(newFilters.length).toEqual(3); + expect(newFilters[0]).toEqual({value: "All", label: "All"}); + expect(newFilters[1]).toEqual({value: "200", label: "200"}); + expect(newFilters[2]).toEqual({value: "400", label: "400"}); + }); +}); diff --git a/src/MlCli/JobApiCall/Job.cs b/src/MlCli/JobApiCall/Job.cs index f0863b85..c11461ee 100644 --- a/src/MlCli/JobApiCall/Job.cs +++ b/src/MlCli/JobApiCall/Job.cs @@ -70,10 +70,10 @@ public async Task ApiCallAsync(Callapi inputTask) var numberChunk = inputTask.ChunkByNumberPart.Value; var chunkIndex = inputTask.ChunkIndex.Value; var d = files.Count / (decimal)numberChunk; - var numberfilesbychunck = d > files.Count / numberChunk + var numberFilesByChunk = d > files.Count / numberChunk ? files.Count / numberChunk + 1 : files.Count / numberChunk; - var listsOfFiles = ChunkBy(files.ToList(), numberfilesbychunck); + var listsOfFiles = ChunkBy(files.ToList(), numberFilesByChunk); files = listsOfFiles[chunkIndex]; } @@ -87,6 +87,7 @@ public async Task ApiCallAsync(Callapi inputTask) var indexFile = 0; var indexFileFetched = 0; var numberKo = 0; + var numberExceptions = 0; while (indexFile < numberFiles) { while (tasks.Count < inputTask.NumberParallel && indexFile < numberFiles) @@ -107,6 +108,11 @@ public async Task ApiCallAsync(Callapi inputTask) numberKo += 1; _logger.LogWarning("number KO: " + numberKo + "/" + (indexFile + 1)); } + else if (task.IsFaulted) + { + numberExceptions += 1; + _logger.LogWarning($"number exceptions: {numberExceptions}/{indexFile + 1}"); + } tasksToRemove.Add(task); } @@ -126,13 +132,12 @@ private async Task PlayDataAsync(HttpClient httpClient, Callapi inputTas string extension, string outputDirectory) { if (Path.GetExtension(currentFilePath) == ".json") return string.Empty; - var jsonFileName = GetTargetFileName(inputTask.IsDefaultTargetFileMode, currentFilePath, extension); var targetFileName = Path.Combine(outputDirectory, jsonFileName); + var fileName = Path.GetFileName(currentFilePath); try { - var fileName = Path.GetFileName(currentFilePath); if (_fileLoader.FileExists(targetFileName)) { _logger.LogWarning($"Task Id: {inputTask.Id} - Already processed file {fileName}"); @@ -148,7 +153,25 @@ private async Task PlayDataAsync(HttpClient httpClient, Callapi inputTas try { httpResult = await CallHttpAsync(httpClient, inputTask, currentFilePath, jsonFileName, i); - if (httpResult.StatusCode < 500) break; + if (httpResult == null) + { + httpResult = new Program.HttpResult + { + FileName = fileName, + FileDirectory = Path.Combine(inputTask.OutputDirectoryJsons, targetFileName), + ImageDirectory = inputTask.OutputDirectoryImages, + FrontDefaultStringsMatcher = inputTask.FrontDefaultStringsMatcher, + StatusCode = 600, + Body = $"Task Id: {inputTask.Id} - Error : http result is null", + Headers = new List>>(), + TimeMs = 0, + Url = inputTask.Url, + TicksAt = DateTime.UtcNow.Ticks, + TryNumber = i + }; + _logger.LogError($"Task Id: {inputTask.Id} - Error : http result is null"); + } + else if (httpResult.StatusCode < 500) break; } catch (Exception e) { @@ -172,9 +195,6 @@ private async Task PlayDataAsync(HttpClient httpClient, Callapi inputTas if (i < inputTask.NumberRetryOnHttp500 + 1) await Task.Delay(inputTask.DelayOn500); } - if (httpResult == null) - throw new ApplicationException("httpResult is null"); - if (httpResult.StatusCode >= 500 && (!inputTask.IsSaveResultOnError || httpResult.StatusCode < 500)) return httpResult.StatusCode < 500 ? "OK" : "KO"; var json = JsonConvert.SerializeObject(httpResult, Formatting.Indented);