Implement FKSearch

impl/c-qube/ingest/config.json

@@ -1,6 +1,7 @@
 {
   "globals": {
-    "onlyCreateWhitelisted": true
+    "onlyCreateWhitelisted": true,
+    "caseSensitiveFKSearch": false
   },
   "dimensions": {
     "namespace": "dimensions",

impl/c-qube/src/services/csv-adapter/parser/utils/csvcleaner.spec.ts

@@ -27,6 +27,7 @@ describe('remove empty lines', () => {
     expect(files).toEqual([
       'test/fixtures/test-csvs/csvcleaner/withEmpty.csv',
       'test/fixtures/test-csvs/csvcleaner/withoutEmpty.csv',
+      "test/fixtures/test-csvs/csvreader/fksearch.csv",
       'test/fixtures/test-csvs/csvreader/invalid.reader.csv',
       'test/fixtures/test-csvs/csvreader/valid.reader.csv',
       'test/fixtures/test-csvs/event-grammars/test-dimension.grammar.csv',

impl/c-qube/src/services/csv-adapter/parser/utils/csvreader.spec.ts

@@ -1,4 +1,4 @@
-import { readCSV, readCSVFile } from './csvreader';
+import { FKvalue, readCSV, readCSVFile } from './csvreader';
 
 describe('CSVReader', () => {
   test('parse the file fine', async () => {
@@ -37,4 +37,22 @@ describe('CSVReader', () => {
   test('benchmarking', () => {
     // add a benchmark test here similar to date parser
   });
-});
+
+  it('should return the correct value from the mocked config file', () => {
+    const configPath = 'ingest/config.json';
+    const result = FKvalue(configPath);
+    expect(result).toEqual(false);
+  });
+
+  it('should correctly process CSV data with case insensitivity', async () => {
+    const filePath = './test/fixtures/test-csvs/csvreader/fksearch.csv';
+    const expectedRows = [
+      ['1', 'john', '30'],
+      ['2', 'david', '25'],
+      ['3', 'michael', '22'],
+      ['4', 'mary', '28'],
+    ];
+    const rows = await readCSV(filePath);
+    expect(rows).toEqual(expectedRows);
+  });
+});

impl/c-qube/src/services/csv-adapter/parser/utils/csvreader.ts

@@ -3,6 +3,7 @@ const fs = require('fs').promises;
 
 import * as csv from 'csv-parser';
 
+const configPath = 'ingest/config.json';
 export async function readCSV(filePath: string): Promise<string[][]> {
   return new Promise((resolve, reject) => {
     const rows: string[][] = [];
@@ -11,7 +12,11 @@ export async function readCSV(filePath: string): Promise<string[][]> {
       .createReadStream(filePath)
       .pipe(csv({ separator: ',', headers: false, quote: "'" }))
       .on('data', (data) => {
-        rows.push(Object.values(data));
+        const rowValues: string[] = Object.values(data);
+        const processedRowValues = FKvalue(configPath)
+          ? rowValues
+          : rowValues.map((value) => value.toLowerCase());
+        rows.push(processedRowValues);
       })
       .on('end', () => {
         resolve(rows);
@@ -30,3 +35,9 @@ export async function readCSVFile(filePath: string): Promise<string[]> {
     .map((row: string) => row.trim())
     .filter((row: string) => row !== '');
 }
+
+export function FKvalue(configPath: string): boolean {
+  const configContent = fs1.readFileSync(configPath, 'utf-8');
+  const config = JSON.parse(configContent);
+  return config.globals.caseSensitiveFKSearch || false ;
+} 

impl/c-qube/test/fixtures/test-csvs/csvreader/fksearch.csv

@@ -0,0 +1,4 @@
+1,John,30
+2,David,25
+3,Michael,22
+4,mary,28