adding ndjson format

jf-tech · Sep 6, 2023 · 455c053 · 455c053
1 parent 79a540b
commit 455c053
Show file tree

Hide file tree

Showing 4 changed files with 28 additions and 6 deletions.
diff --git a/cli/cmd/transformCmd.go b/cli/cmd/transformCmd.go
@@ -86,6 +86,11 @@ func doTransform() error {
 		if err != nil {
 			return "", err
 		}
+
+		if schema.Header().ParserSettings.NDJSON {
+			return string(b), nil
+		}
+
 		return strings.Join(
 			strs.NoErrMapSlice(
 				strings.Split(jsons.BPJ(string(b)), "\n"),
@@ -95,13 +100,27 @@ func doTransform() error {
 
 	record, err := doOne()
 	if err == io.EOF {
-		fmt.Println("[]")
+		if schema.Header().ParserSettings.NDJSON {
+			fmt.Println("")
+		} else {
+			fmt.Println("[]")
+		}
 		return nil
 	}
 	if err != nil {
 		return err
 	}
-	fmt.Printf("[\n%s", record)
+
+	start := "[\n%s"
+	middle := ",\n%s"
+	end := "\n]"
+	if schema.Header().ParserSettings.NDJSON {
+		start = "%s"
+		middle = "\n%s"
+		end = ""
+	}
+
+	fmt.Printf(start, record)
 	for {
 		record, err = doOne()
 		if err == io.EOF {
@@ -110,8 +129,8 @@ func doTransform() error {
 		if err != nil {
 			return err
 		}
-		fmt.Printf(",\n%s", record)
+		fmt.Printf(middle, record)
 	}
-	fmt.Println("\n]")
+	fmt.Println(end)
 	return nil
 }
diff --git a/header/header.go b/header/header.go
@@ -15,6 +15,7 @@ type ParserSettings struct {
 	Version        string  `json:"version,omitempty"`
 	FileFormatType string  `json:"file_format_type,omitempty"`
 	Encoding       *string `json:"encoding,omitempty"`
+	NDJSON         bool    `json:"ndjson,omitempty"`
 }
 
 const (

diff --git a/validation/parserSettings.go b/validation/parserSettings.go
diff --git a/validation/parserSettings.json b/validation/parserSettings.json
@@ -12,7 +12,8 @@
                 "encoding": {
                     "type": "string",
                     "enum": [ "utf-8", "iso-8859-1", "windows-1252" ]
-                }
+                },
+                "ndjson": { "type": "boolean" }
             },
             "required": [ "version", "file_format_type" ],
             "additionalProperties": false