Skip to content

Commit

Permalink
fix(ApiCall): prevent ML-CLI from generating null results for api call (
Browse files Browse the repository at this point in the history 
#299)
  • Loading branch information
@lilian-delouvy
lilian-delouvy authored Dec 22, 2023
1 parent f324425 commit 630bd8d
Show file tree
Hide file tree
Showing 3 changed files with 219 additions and 90 deletions.
168 changes: 86 additions & 82 deletions src/Ecotag/ClientApp/src/Local/FileTreatment/FileTreatment.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,80 @@ const timeSideVariables = [
{value: 'Right', label: 'Right'}
];

const FileTreatment = ({state, setState, MonacoEditor, fetch}) => {
const checkboxOptions = [
{
key: "checkbox_isAnnotationOpen",
id: "is_annotation_open_checkbox",
value: "test",
label: "Is annotation open ? ",
disabled: false
}
];

export const mapItems = data => {
const mappedItems = data.map(item => {
if (!item || !item.Left || !item.Right) {
if(item){
console.log("File " + item.FileName + " does not contain the required data, and will be ignored. Please investigate on this file.");
console.log(item);
}
return;
}
return {
fileName: item.FileName,
left: {
Url: item.Left.Url,
FileName: item.Left.FileName,
FileDirectory: item.Left.FileDirectory,
ImageDirectory: item.Left.ImageDirectory,
FrontDefaultStringsMatcher: item.Left.FrontDefaultStringsMatcher,
StatusCode: item.Left.StatusCode,
Body: item.Left.Body,
Headers: item.Left.Headers,
TimeMs: item.Left.TimeMs,
TicksAt: item.Left.TicksAt
},
right: {
Url: item.Right.Url,
FileName: item.Right.FileName,
FileDirectory: item.Right.FileDirectory,
ImageDirectory: item.Right.ImageDirectory,
FrontDefaultStringsMatcher: item.Right.FrontDefaultStringsMatcher,
StatusCode: item.Right.StatusCode,
Body: item.Right.Body,
Headers: item.Right.Headers,
TimeMs: item.Right.TimeMs,
TicksAt: item.Right.TicksAt
},
id: cuid(),
parse: false,
collapse: true,
};
});
return mappedItems.filter(item => item !== undefined && item !== null);
};

export const compareStatusCode = (status, result) => {
const strStatus = status.toString();
const index = result.findIndex(element => element.value === strStatus);
if (index !== -1) {
return;
}
const newItem = {value: strStatus, label: strStatus};
result.push(newItem);
};

export const setStatusFilterItems = newItems => {
const allCodes = {value: "All", label: "All"};
const result = [allCodes];
newItems.forEach(item => {
compareStatusCode(item.left.StatusCode, result);
compareStatusCode(item.right.StatusCode, result);
});
return result;
};

const FileTreatment = ({state, setState, MonacoEditor, fetch}) => {
const [filterState, setFilterState] = useState({
filterName: "KO",
extensionName: "All",
Expand Down Expand Up @@ -71,90 +143,22 @@ try {
}`
});

const checkboxOptions = [
{
key: "checkbox_isAnnotationOpen",
id: "is_annotation_open_checkbox",
value: "test",
label: "Is annotation open ? ",
disabled: false
}
];

const mapItems = data => (data.map(item => {
if (item.Left == null || item.Right == null) {
return;
}
return {
fileName: item.FileName,
left: {
Url: item.Left.Url,
FileName: item.Left.FileName,
FileDirectory: item.Left.FileDirectory,
ImageDirectory: item.Left.ImageDirectory,
FrontDefaultStringsMatcher: item.Left.FrontDefaultStringsMatcher,
StatusCode: item.Left.StatusCode,
Body: item.Left.Body,
Headers: item.Left.Headers,
TimeMs: item.Left.TimeMs,
TicksAt: item.Left.TicksAt
},
right: {
Url: item.Right.Url,
FileName: item.Right.FileName,
FileDirectory: item.Right.FileDirectory,
ImageDirectory: item.Right.ImageDirectory,
FrontDefaultStringsMatcher: item.Right.FrontDefaultStringsMatcher,
StatusCode: item.Right.StatusCode,
Body: item.Right.Body,
Headers: item.Right.Headers,
TimeMs: item.Right.TimeMs,
TicksAt: item.Right.TicksAt
},
id: cuid(),
parse: false,
collapse: true,
};
}));

const compareStatusCode = (status, result) => {
const strStatus = status.toString();
const index = result.findIndex(element => element.value === strStatus);
if (index !== -1) {
return;
}
const newItem = {value: strStatus, label: strStatus};
result.push(newItem);
};

const setStatusFilterItems = newItems => {
const allCodes = {value: "All", label: "All"};
const result = [allCodes];
newItems.forEach(item => {
compareStatusCode(item.left.StatusCode, result);
compareStatusCode(item.right.StatusCode, result);
});
return result;
};

const onLoad = (result, fileName) => {
const isVersion0 = Array.isArray(result);
if (!result.hasOwnProperty('CompareLocation')) {
if(!result || !Array.isArray(result.Content) || !result.CompareLocation){
onLoadFailure(fileName);
} else {
const location = isVersion0 ? "" : result.CompareLocation;
const mappedItems = mapItems(isVersion0 ? result : result.Content);
const statusCodeItems = setStatusFilterItems(mappedItems);
setState
({
...state,
fileName: fileName,
compareLocation: location,
items: mappedItems,
statusCodes: statusCodeItems
});
setFilterState({...filterState, loadFileError: false});
}
const location = result.CompareLocation;
const mappedItems = mapItems(result.Content);
const statusCodeItems = setStatusFilterItems(mappedItems);
setState
({
...state,
fileName: fileName,
compareLocation: location,
items: mappedItems,
statusCodes: statusCodeItems
});
setFilterState({...filterState, loadFileError: false});
};

const onLoadFailure = fileName => {
Expand Down
105 changes: 105 additions & 0 deletions src/Ecotag/ClientApp/src/Local/FileTreatment/FileTreatment.spec.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
import {compareStatusCode, mapItems, setStatusFilterItems,} from './FileTreatment';

describe('Check file treatment', () => {
it('should map items properly', () => {
const data = [
{
FileName: "MyFirstFileName.txt",
Left: {
Url: "http://localhost:5000",
FileName: "MyFirstFileName.txt",
FileDirectory: "folder1",
ImageDirectory: "folder2",
FrontDefaultStringsMatcher: "",
StatusCode: 200,
Body: "Blah blah blah",
Headers: "",
TimeMs: 4000,
TicksAt: 0
},
Right: {
Url: "http://localhost:5000",
FileName: "MyFirstFileName.txt",
FileDirectory: "folder1",
ImageDirectory: "folder2",
FrontDefaultStringsMatcher: "",
StatusCode: 400,
Body: "Blah blah blah",
Headers: "",
TimeMs: 4000,
TicksAt: 0
}
},
undefined,
{
FileName: "MySecondFileName.txt",
Left: {
Url: "http://localhost:5000",
FileName: "MySecondFileName.txt",
FileDirectory: "folder1",
ImageDirectory: "folder2",
FrontDefaultStringsMatcher: "",
StatusCode: 200,
Body: "Blah blah blah",
Headers: "",
TimeMs: 4000,
TicksAt: 0
},
Right: {
Url: "http://localhost:5000",
FileName: "MySecondFileName.txt",
FileDirectory: "folder1",
ImageDirectory: "folder2",
FrontDefaultStringsMatcher: "",
StatusCode: 400,
Body: "Blah blah blah",
Headers: "",
TimeMs: 4000,
TicksAt: 0
}
},
null
];
const mappedData = mapItems(data);
expect(mappedData.length).toEqual(2);
for(let item in mappedData){
expect(item).not.toBeUndefined();
expect(item).not.toBeNull();
}
});
it('should add new status code properly', () => {
const statusCode = 200;
const result = [{value: "All", label: "All"}];
compareStatusCode(statusCode, result);

expect(result.length).toEqual(2);
expect(result[0]).toEqual({value: "All", label: "All"});
expect(result[1]).toEqual({value: "200", label: "200"});
});
it('should set status filter items properly', () => {
const items = [
{
left: {
StatusCode: 200
},
right: {
StatusCode: 200
}
},
{
left: {
StatusCode: 200
},
right: {
StatusCode: 400
}
}
];
const newFilters = setStatusFilterItems(items);

expect(newFilters.length).toEqual(3);
expect(newFilters[0]).toEqual({value: "All", label: "All"});
expect(newFilters[1]).toEqual({value: "200", label: "200"});
expect(newFilters[2]).toEqual({value: "400", label: "400"});
});
});
36 changes: 28 additions & 8 deletions src/MlCli/JobApiCall/Job.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,10 +70,10 @@ public async Task ApiCallAsync(Callapi inputTask)
var numberChunk = inputTask.ChunkByNumberPart.Value;
var chunkIndex = inputTask.ChunkIndex.Value;
var d = files.Count / (decimal)numberChunk;
var numberfilesbychunck = d > files.Count / numberChunk
var numberFilesByChunk = d > files.Count / numberChunk
? files.Count / numberChunk + 1
: files.Count / numberChunk;
var listsOfFiles = ChunkBy(files.ToList(), numberfilesbychunck);
var listsOfFiles = ChunkBy(files.ToList(), numberFilesByChunk);
files = listsOfFiles[chunkIndex];
}

Expand All @@ -87,6 +87,7 @@ public async Task ApiCallAsync(Callapi inputTask)
var indexFile = 0;
var indexFileFetched = 0;
var numberKo = 0;
var numberExceptions = 0;
while (indexFile < numberFiles)
{
while (tasks.Count < inputTask.NumberParallel && indexFile < numberFiles)
Expand All @@ -107,6 +108,11 @@ public async Task ApiCallAsync(Callapi inputTask)
numberKo += 1;
_logger.LogWarning("number KO: " + numberKo + "/" + (indexFile + 1));
}
else if (task.IsFaulted)
{
numberExceptions += 1;
_logger.LogWarning($"number exceptions: {numberExceptions}/{indexFile + 1}");
}

tasksToRemove.Add(task);
}
Expand All @@ -126,13 +132,12 @@ private async Task<string> PlayDataAsync(HttpClient httpClient, Callapi inputTas
string extension, string outputDirectory)
{
if (Path.GetExtension(currentFilePath) == ".json") return string.Empty;


var jsonFileName = GetTargetFileName(inputTask.IsDefaultTargetFileMode, currentFilePath, extension);
var targetFileName = Path.Combine(outputDirectory, jsonFileName);
var fileName = Path.GetFileName(currentFilePath);
try
{
var fileName = Path.GetFileName(currentFilePath);
if (_fileLoader.FileExists(targetFileName))
{
_logger.LogWarning($"Task Id: {inputTask.Id} - Already processed file {fileName}");
Expand All @@ -148,7 +153,25 @@ private async Task<string> PlayDataAsync(HttpClient httpClient, Callapi inputTas
try
{
httpResult = await CallHttpAsync(httpClient, inputTask, currentFilePath, jsonFileName, i);
if (httpResult.StatusCode < 500) break;
if (httpResult == null)
{
httpResult = new Program.HttpResult
{
FileName = fileName,
FileDirectory = Path.Combine(inputTask.OutputDirectoryJsons, targetFileName),
ImageDirectory = inputTask.OutputDirectoryImages,
FrontDefaultStringsMatcher = inputTask.FrontDefaultStringsMatcher,
StatusCode = 600,
Body = $"Task Id: {inputTask.Id} - Error : http result is null",
Headers = new List<KeyValuePair<string, IEnumerable<string>>>(),
TimeMs = 0,
Url = inputTask.Url,
TicksAt = DateTime.UtcNow.Ticks,
TryNumber = i
};
_logger.LogError($"Task Id: {inputTask.Id} - Error : http result is null");
}
else if (httpResult.StatusCode < 500) break;
}
catch (Exception e)
{
Expand All @@ -172,9 +195,6 @@ private async Task<string> PlayDataAsync(HttpClient httpClient, Callapi inputTas
if (i < inputTask.NumberRetryOnHttp500 + 1) await Task.Delay(inputTask.DelayOn500);
}

if (httpResult == null)
throw new ApplicationException("httpResult is null");

if (httpResult.StatusCode >= 500 && (!inputTask.IsSaveResultOnError || httpResult.StatusCode < 500))
return httpResult.StatusCode < 500 ? "OK" : "KO";
var json = JsonConvert.SerializeObject(httpResult, Formatting.Indented);
Expand Down

0 comments on commit 630bd8d

Please sign in to comment.