From a0952623749075997f67e7afb0b4eca9b4e4e4e9 Mon Sep 17 00:00:00 2001 From: Tom Reitz Date: Tue, 18 Jun 2024 10:54:06 -0500 Subject: [PATCH 1/6] fix nested json not working when rendering destination templates --- earthmover/nodes/destination.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/earthmover/nodes/destination.py b/earthmover/nodes/destination.py index 5199beab..32b7f661 100644 --- a/earthmover/nodes/destination.py +++ b/earthmover/nodes/destination.py @@ -1,5 +1,6 @@ import os import pandas as pd +import json import re from earthmover.nodes.node import Node @@ -113,8 +114,11 @@ def execute(self, **kwargs): self.size = os.path.getsize(self.file) def render_row(self, row: pd.Series): - row = row.astype("string").fillna('') - _data_tuple = row.to_dict() + types_to_cast = [bool, int, float] + cols_to_cast = list(filter(lambda x: type(row[x]) in types_to_cast, row.keys())) + row = row.to_dict() + row = dict(map(lambda x: (x[0], str(x[1]) if x[0] in cols_to_cast else (x[1] if x[1] else "")), row.items())) + _data_tuple = row _data_tuple["__row_data__"] = row try: From 09a0227bcfd783a751574275e5dd858c41615699 Mon Sep 17 00:00:00 2001 From: Tom Reitz Date: Tue, 18 Jun 2024 10:59:34 -0500 Subject: [PATCH 2/6] update cols to keys for dict --- earthmover/nodes/destination.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/earthmover/nodes/destination.py b/earthmover/nodes/destination.py index 32b7f661..15553701 100644 --- a/earthmover/nodes/destination.py +++ b/earthmover/nodes/destination.py @@ -115,9 +115,9 @@ def execute(self, **kwargs): def render_row(self, row: pd.Series): types_to_cast = [bool, int, float] - cols_to_cast = list(filter(lambda x: type(row[x]) in types_to_cast, row.keys())) row = row.to_dict() - row = dict(map(lambda x: (x[0], str(x[1]) if x[0] in cols_to_cast else (x[1] if x[1] else "")), row.items())) + keys_to_cast = list(filter(lambda x: type(row[x]) in types_to_cast, row.keys())) + row = dict(map(lambda x: (x[0], str(x[1]) if x[0] in keys_to_cast else (x[1] if x[1] else "")), row.items())) _data_tuple = row _data_tuple["__row_data__"] = row From 0dd8a6383b2fa543b824a8c2eb021738c733fdbb Mon Sep 17 00:00:00 2001 From: Tom Reitz Date: Tue, 18 Jun 2024 11:00:02 -0500 Subject: [PATCH 3/6] remove json import --- earthmover/nodes/destination.py | 1 - 1 file changed, 1 deletion(-) diff --git a/earthmover/nodes/destination.py b/earthmover/nodes/destination.py index 15553701..31900513 100644 --- a/earthmover/nodes/destination.py +++ b/earthmover/nodes/destination.py @@ -1,6 +1,5 @@ import os import pandas as pd -import json import re from earthmover.nodes.node import Node From 15fa212ee80b5b1a2c62896c6dde8427c4e8b430 Mon Sep 17 00:00:00 2001 From: Tom Reitz Date: Tue, 18 Jun 2024 11:00:43 -0500 Subject: [PATCH 4/6] tweak --- earthmover/nodes/destination.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/earthmover/nodes/destination.py b/earthmover/nodes/destination.py index 31900513..91dcaab5 100644 --- a/earthmover/nodes/destination.py +++ b/earthmover/nodes/destination.py @@ -113,8 +113,8 @@ def execute(self, **kwargs): self.size = os.path.getsize(self.file) def render_row(self, row: pd.Series): - types_to_cast = [bool, int, float] row = row.to_dict() + types_to_cast = [bool, int, float] keys_to_cast = list(filter(lambda x: type(row[x]) in types_to_cast, row.keys())) row = dict(map(lambda x: (x[0], str(x[1]) if x[0] in keys_to_cast else (x[1] if x[1] else "")), row.items())) _data_tuple = row From bf3ddd995a8617e6cc3c6d746032830ae25d1638 Mon Sep 17 00:00:00 2001 From: Tom Reitz Date: Tue, 18 Jun 2024 11:04:01 -0500 Subject: [PATCH 5/6] update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b724eaf3..a728ae88 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ Released 2024-06-18 * hotfix: Resolve incompatible package dependencies +* hotfix: Fix type casting of nested JSON for destination templates From f36db9a7f127e8b80198773edb903a82e27749d7 Mon Sep 17 00:00:00 2001 From: Tom Reitz Date: Tue, 18 Jun 2024 11:12:32 -0500 Subject: [PATCH 6/6] add comment --- earthmover/nodes/destination.py | 1 + 1 file changed, 1 insertion(+) diff --git a/earthmover/nodes/destination.py b/earthmover/nodes/destination.py index 91dcaab5..94e10037 100644 --- a/earthmover/nodes/destination.py +++ b/earthmover/nodes/destination.py @@ -116,6 +116,7 @@ def render_row(self, row: pd.Series): row = row.to_dict() types_to_cast = [bool, int, float] keys_to_cast = list(filter(lambda x: type(row[x]) in types_to_cast, row.keys())) + # this line (a) converts the keys_to_cast to string, and also converts all Nones to empty string: row = dict(map(lambda x: (x[0], str(x[1]) if x[0] in keys_to_cast else (x[1] if x[1] else "")), row.items())) _data_tuple = row _data_tuple["__row_data__"] = row