diff --git a/gramps/gen/db/conversion_tools/__init__.py b/gramps/gen/db/conversion_tools/__init__.py new file mode 100644 index 00000000000..0743cd6d781 --- /dev/null +++ b/gramps/gen/db/conversion_tools/__init__.py @@ -0,0 +1,21 @@ +# +# Gramps - a GTK+/GNOME based genealogy program +# +# Copyright (C) 2024 Doug Blank +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + +from .conversion_21 import convert_21 diff --git a/gramps/gen/db/conversion_tools/conversion_21.py b/gramps/gen/db/conversion_tools/conversion_21.py new file mode 100644 index 00000000000..47af3f09711 --- /dev/null +++ b/gramps/gen/db/conversion_tools/conversion_21.py @@ -0,0 +1,493 @@ +# +# Gramps - a GTK+/GNOME based genealogy program +# +# Copyright (C) 2024 Doug Blank +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + + +def convert_21(classname, array): + if classname == "Person": + return convert_person(array) + elif classname == "Family": + return convert_family(array) + elif classname == "Event": + return convert_event(array) + elif classname == "Place": + return convert_place(array) + elif classname == "Repository": + return convert_repository(array) + elif classname == "Source": + return convert_source(array) + elif classname == "Citation": + return convert_citation(array) + elif classname == "Media": + return convert_media(array) + elif classname == "Note": + return convert_note(array) + elif classname == "Tag": + return convert_tag(array) + else: + raise Exception("unknown class: %s" % classname) + + +def convert_researcher(researcher): + # This can be a problem if the Researcher class + # changes! + return { + "_class": "Researcher", + "street": researcher.street, + "city": researcher.city, + "county": researcher.county, + "state": researcher.state, + "country": researcher.country, + "postal": researcher.postal, + "phone": researcher.phone, + "locality": researcher.locality, + "name": researcher.name, + "addr": researcher.addr, + "email": researcher.email, + } + + +def convert_person(array): + return { + "_class": "Person", + "handle": array[0], + "gramps_id": array[1], + "gender": array[2], + "primary_name": convert_name(array[3]), + "alternate_names": [convert_name(name) for name in array[4]], + "death_ref_index": array[5], + "birth_ref_index": array[6], + "event_ref_list": [convert_event_ref(ref) for ref in array[7]], + "family_list": array[8], + "parent_family_list": array[9], + "media_list": [convert_media_ref(ref) for ref in array[10]], + "address_list": [convert_address(address) for address in array[11]], + "attribute_list": [convert_attribute("Attribute", attr) for attr in array[12]], + "urls": [convert_url(url) for url in array[13]], + "lds_ord_list": [convert_ord(ord) for ord in array[14]], + "citation_list": array[15], # handles + "note_list": array[16], # handles + "change": array[17], + "tag_list": array[18], # handles + "private": array[19], + "person_ref_list": [convert_person_ref(ref) for ref in array[20]], + } + + +def convert_person_ref(array): + return { + "_class": "PersonRef", + "private": array[0], + "citation_list": array[1], # handles + "note_list": array[2], # handles + "ref": array[3], + "rel": array[4], + } + + +def convert_ord(array): + return { + "_class": "LdsOrd", + "citation_list": array[0], # handles + "note_list": array[1], # handles + "date": convert_date(array[2]), + "type": array[3], # string + "place": array[4], + "famc": array[5], + "temple": array[6], + "status": array[7], + "private": array[8], + } + + +def convert_url(array): + return { + "_class": "Url", + "private": array[0], + "path": array[1], + "desc": array[2], + "type": convert_type("UrlType", array[3]), + } + + +def convert_address(array): + return { + "_class": "Address", + "private": array[0], + "citation_list": array[1], # handles + "note_list": array[2], # handles + "date": convert_date(array[3]), + "street": array[4][0], + "locality": array[4][1], + "city": array[4][2], + "county": array[4][3], + "state": array[4][4], + "country": array[4][5], + "postal": array[4][6], + "phone": array[4][7], + } + + +def convert_media_ref(array): + return { + "_class": "MediaRef", + "private": array[0], + "citation_list": array[1], # handles + "note_list": array[2], # handles + "attribute_list": [convert_attribute("Attribute", attr) for attr in array[3]], + "ref": array[4], + "rect": list(array[5]) if array[5] is not None else None, + } + + +def convert_event_ref(array): + return { + "_class": "EventRef", + "private": array[0], + "citation_list": array[1], # handles + "note_list": array[2], # handles + "attribute_list": [convert_attribute("Attribute", attr) for attr in array[3]], + "ref": array[4], + "role": convert_type("EventRoleType", array[5]), + } + + +def convert_attribute(classname, array): + if classname == "SrcAttribute": + return { + "_class": classname, + "private": array[0], + "type": convert_type("SrcAttributeType", array[1]), + "value": array[2], + } + else: + return { + "_class": classname, + "private": array[0], + "citation_list": array[1], # handles + "note_list": array[2], # handles + "type": convert_type("AttributeType", array[3]), + "value": array[4], + } + + +def convert_name(array): + return { + "_class": "Name", + "private": array[0], + "citation_list": array[1], + "note_list": array[2], # handles + "date": convert_date(array[3]), + "first_name": array[4], + "surname_list": [convert_surname(name) for name in array[5]], + "suffix": array[6], + "title": array[7], + "type": convert_type("NameType", array[8]), + "group_as": array[9], + "sort_as": array[10], + "display_as": array[11], + "call": array[12], + "nick": array[13], + "famnick": array[14], + } + + +def convert_surname(array): + return { + "_class": "Surname", + "surname": array[0], + "prefix": array[1], + "primary": array[2], + "origintype": convert_type("NameOriginType", array[3]), + "connector": array[4], + } + + +def convert_date(array): + if array is None: + return { + "_class": "Date", + "calendar": 0, + "modifier": 0, + "quality": 0, + "dateval": [0, 0, 0, False], + "text": "", + "sortval": 0, + "newyear": 0, + "format": None, + } + else: + return { + "_class": "Date", + "calendar": array[0], + "modifier": array[1], + "quality": array[2], + "dateval": list(array[3]), + "text": array[4], + "sortval": array[5], + "newyear": array[6], + "format": None, + } + + +def convert_stt(array): + return { + "_class": "StyledTextTag", + "name": convert_type("StyledTextTagType", array[0]), + "value": array[1], + "ranges": [list(r) for r in array[2]], + } + + +def convert_type(classname, array): + return { + "_class": classname, + "value": array[0], + "string": array[1], + } + + +def convert_family(array): + return { + "_class": "Family", + "handle": array[0], + "gramps_id": array[1], + "father_handle": array[2], + "mother_handle": array[3], + "child_ref_list": [convert_child_ref(ref) for ref in array[4]], + "type": convert_type("FamilyRelType", array[5]), + "event_ref_list": [convert_event_ref(ref) for ref in array[6]], + "media_list": [convert_media_ref(ref) for ref in array[7]], + "attribute_list": [convert_attribute("Attribute", attr) for attr in array[8]], + "lds_ord_list": [convert_ord(ord) for ord in array[9]], + "citation_list": array[10], # handles + "note_list": array[11], # handles + "change": array[12], + "tag_list": array[13], + "private": array[14], + "complete": 0, + } + + +def convert_child_ref(array): + return { + "_class": "ChildRef", + "private": array[0], + "citation_list": array[1], + "note_list": array[2], + "ref": array[3], + "frel": convert_type("ChildRefType", array[4]), + "mrel": convert_type("ChildRefType", array[5]), + } + + +def convert_event(array): + return { + "_class": "Event", + "handle": array[0], + "gramps_id": array[1], + "type": convert_type("EventType", array[2]), + "date": convert_date(array[3]), + "description": array[4], + "place": array[5], + "citation_list": array[6], + "note_list": array[7], + "media_list": [convert_media_ref(ref) for ref in array[8]], + "attribute_list": [convert_attribute("Attribute", attr) for attr in array[9]], + "change": array[10], + "tag_list": array[11], + "private": array[12], + } + + +def convert_place(array): + return { + "_class": "Place", + "handle": array[0], + "gramps_id": array[1], + "title": array[2], + "long": array[3], + "lat": array[4], + "placeref_list": [convert_place_ref(ref) for ref in array[5]], + "name": convert_place_name(array[6]), + "alt_names": [convert_place_name(name) for name in array[7]], + "place_type": convert_type("PlaceType", array[8]), + "code": array[9], + "alt_loc": [convert_location(loc) for loc in array[10]], + "urls": [convert_url(url) for url in array[11]], + "media_list": [convert_media_ref(ref) for ref in array[12]], + "citation_list": array[13], + "note_list": array[14], + "change": array[15], + "tag_list": array[16], + "private": array[17], + } + + +def convert_location(array): + return { + "_class": "Location", + "street": array[0][0], + "locality": array[0][1], + "city": array[0][2], + "county": array[0][3], + "state": array[0][4], + "country": array[0][5], + "postal": array[0][6], + "phone": array[0][7], + "parish": array[1], + } + + +def convert_place_ref(array): + return { + "_class": "PlaceRef", + "ref": array[0], + "date": convert_date(array[1]), + } + + +def convert_place_name(array): + return { + "_class": "PlaceName", + "value": array[0], + "date": convert_date(array[1]), + "lang": array[2], + } + + +def convert_repository(array): + return { + "_class": "Repository", + "handle": array[0], + "gramps_id": array[1], + "type": convert_type("RepositoryType", array[2]), + "name": array[3], + "note_list": array[4], + "address_list": [convert_address(addr) for addr in array[5]], + "urls": [convert_url(url) for url in array[6]], + "change": array[7], + "tag_list": array[8], + "private": array[9], + } + + +def convert_source(array): + return { + "_class": "Source", + "handle": array[0], + "gramps_id": array[1], + "title": array[2], + "author": array[3], + "pubinfo": array[4], + "note_list": array[5], + "media_list": [convert_media_ref(ref) for ref in array[6]], + "abbrev": array[7], + "change": array[8], + "attribute_list": [ + convert_attribute("SrcAttribute", attr) for attr in array[9] + ], + "reporef_list": [convert_repo_ref(ref) for ref in array[10]], + "tag_list": array[11], + "private": array[12], + } + + +def convert_repo_ref(array): + return { + "_class": "RepoRef", + "note_list": array[0], + "ref": array[1], + "call_number": array[2], + "media_type": convert_type("SourceMediaType", array[3]), + "private": array[4], + } + + +def convert_citation(array): + return { + "_class": "Citation", + "handle": array[0], + "gramps_id": array[1], + "date": convert_date(array[2]), + "page": array[3], + "confidence": array[4], + "source_handle": array[5], + "note_list": array[6], + "media_list": [convert_media_ref(ref) for ref in array[7]], + "attribute_list": [ + convert_attribute("SrcAttribute", attr) for attr in array[8] + ], + "change": array[9], + "tag_list": array[10], + "private": array[11], + } + + +def convert_media(array): + return { + "_class": "Media", + "handle": array[0], + "gramps_id": array[1], + "path": array[2], + "mime": array[3], + "desc": array[4], + "checksum": array[5], + "attribute_list": [convert_attribute("Attribute", attr) for attr in array[6]], + "citation_list": array[7], + "note_list": array[8], + "change": array[9], + "date": convert_date(array[10]), + "tag_list": array[11], + "private": array[12], + "thumb": None, + } + + +def convert_note(array): + return { + "_class": "Note", + "handle": array[0], + "gramps_id": array[1], + "text": convert_styledtext(array[2]), + "format": array[3], + "type": convert_type("NoteType", array[4]), + "change": array[5], + "tag_list": array[6], + "private": array[7], + } + + +def convert_styledtext(array): + return { + "_class": "StyledText", + "string": array[0], + "tags": [convert_stt(stt) for stt in array[1]], + } + + +def convert_tag(array): + return { + "_class": "Tag", + "handle": array[0], + "name": array[1], + "color": array[2], + "priority": array[3], + "change": array[4], + } diff --git a/gramps/gen/db/generic.py b/gramps/gen/db/generic.py index 0c51128eb2c..309fbc97da7 100644 --- a/gramps/gen/db/generic.py +++ b/gramps/gen/db/generic.py @@ -3,6 +3,7 @@ # # Copyright (C) 2015-2016 Gramps Development Team # Copyright (C) 2016 Nick Hall +# Copyright (C) 2024 Doug Blank # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -57,6 +58,7 @@ Source, Tag, ) +from ..lib.serialize import from_dict, BlobSerializer, JSONSerializer from ..lib.genderstats import GenderStats from ..lib.researcher import Researcher from ..updatecallback import UpdateCallback @@ -228,7 +230,7 @@ def _undo(self, update_history): try: self.db._txn_begin() for record_id in subitems: - (key, trans_type, handle, old_data, _) = pickle.loads( + (key, trans_type, handle, old_data, x) = pickle.loads( self.undodb[record_id] ) @@ -389,7 +391,7 @@ class DbGeneric(DbWriteBase, DbReadBase, UpdateCallback, Callback): __callback_map = {} - VERSION = (20, 0, 0) + VERSION = (21, 0, 0) def __init__(self, directory=None): DbReadBase.__init__(self) @@ -621,6 +623,18 @@ def _initialize(self, directory, username, password): """ raise NotImplementedError + def upgrade_table_for_json_data(self, table_name): + """ + Overload this method to upgrade the table to store data in JSON format + """ + raise NotImplementedError + + def use_json_data(self): + """ + Overload this method to check if the database stores objects in JSON format + """ + raise NotImplementedError + def __check_readonly(self, name): """ Return True if we don't have read/write access to the database, @@ -669,8 +683,17 @@ def load( # run backend-specific code: self._initialize(directory, username, password) + need_to_set_version = False if not self._schema_exists(): self._create_schema() + need_to_set_version = True + + if self.use_json_data(): + self.set_serializer("json") + else: + self.set_serializer("blob") + + if need_to_set_version: self._set_metadata("version", str(self.VERSION[0])) # Load metadata @@ -900,6 +923,13 @@ def transaction_begin(self, transaction): self.transaction = transaction return transaction + def _get_metadata_keys(self): + """ + Get all of the metadata setting names from the + database. + """ + raise NotImplementedError + def _get_metadata(self, key, default=[]): """ Get an item from the database. @@ -912,10 +942,12 @@ def _get_metadata(self, key, default=[]): """ raise NotImplementedError - def _set_metadata(self, key, value): + def _set_metadata(self, key, value, use_txn=True): """ key: string value: item, will be serialized here + + Note: if use_txn, then begin/commit txn """ raise NotImplementedError @@ -1355,7 +1387,8 @@ def _get_from_handle(self, obj_key, obj_class, handle): raise HandleError("Handle is empty") data = self._get_raw_data(obj_key, handle) if data: - return obj_class.create(data) + return self.serializer.data_to_object(obj_class, data) + raise HandleError(f"Handle {handle} not found") def get_event_from_handle(self, handle): @@ -1396,39 +1429,39 @@ def get_tag_from_handle(self, handle): def get_person_from_gramps_id(self, gramps_id): data = self._get_raw_person_from_id_data(gramps_id) - return Person.create(data) + return self.serializer.data_to_object(Person, data) def get_family_from_gramps_id(self, gramps_id): data = self._get_raw_family_from_id_data(gramps_id) - return Family.create(data) + return self.serializer.data_to_object(Family, data) def get_citation_from_gramps_id(self, gramps_id): data = self._get_raw_citation_from_id_data(gramps_id) - return Citation.create(data) + return self.serializer.data_to_object(Citation, data) def get_source_from_gramps_id(self, gramps_id): data = self._get_raw_source_from_id_data(gramps_id) - return Source.create(data) + return self.serializer.data_to_object(Source, data) def get_event_from_gramps_id(self, gramps_id): data = self._get_raw_event_from_id_data(gramps_id) - return Event.create(data) + return self.serializer.data_to_object(Event, data) def get_media_from_gramps_id(self, gramps_id): data = self._get_raw_media_from_id_data(gramps_id) - return Media.create(data) + return self.serializer.data_to_object(Media, data) def get_place_from_gramps_id(self, gramps_id): data = self._get_raw_place_from_id_data(gramps_id) - return Place.create(data) + return self.serializer.data_to_object(Place, data) def get_repository_from_gramps_id(self, gramps_id): data = self._get_raw_repository_from_id_data(gramps_id) - return Repository.create(data) + return self.serializer.data_to_object(Repository, data) def get_note_from_gramps_id(self, gramps_id): data = self._get_raw_note_from_id_data(gramps_id) - return Note.create(data) + return self.serializer.data_to_object(Note, data) ################################################################ # @@ -1629,7 +1662,7 @@ def _iter_objects(self, class_): """ cursor = self._get_table_func(class_.__name__, "cursor_func") for data in cursor(): - yield class_.create(data[1]) + yield self.serializer.data_to_object(class_, data[1]) def iter_people(self): return self._iter_objects(Person) @@ -1744,7 +1777,7 @@ def _iter_raw_place_tree_data(self): def _get_raw_data(self, obj_key, handle): """ - Return raw (serialized and pickled) object from handle. + Return raw (serialized) object from handle. """ raise NotImplementedError @@ -1935,7 +1968,7 @@ def commit_person(self, person, transaction, change_time=None): old_data = self._commit_base(person, PERSON_KEY, transaction, change_time) if old_data: - old_person = Person(old_data) + old_person = from_dict(old_data) # Update gender statistics if necessary if old_person.gender != person.gender or ( old_person.primary_name.first_name != person.primary_name.first_name @@ -2663,6 +2696,7 @@ def _gramps_upgrade(self, version, directory, callback=None): gramps_upgrade_18, gramps_upgrade_19, gramps_upgrade_20, + gramps_upgrade_21, ) if version < 14: @@ -2679,6 +2713,8 @@ def _gramps_upgrade(self, version, directory, callback=None): gramps_upgrade_19(self) if version < 20: gramps_upgrade_20(self) + if version < 21: + gramps_upgrade_21(self) self.rebuild_secondary(callback) self.reindex_reference_map(callback) @@ -2694,3 +2730,12 @@ def get_schema_version(self): def set_schema_version(self, value): """set the current schema version""" self._set_metadata("version", str(value)) + + def set_serializer(self, serializer_name): + """ + Set the serializer to 'blob' or 'json' + """ + if serializer_name == "blob": + self.serializer = BlobSerializer + elif serializer_name == "json": + self.serializer = JSONSerializer diff --git a/gramps/gen/db/upgrade.py b/gramps/gen/db/upgrade.py index 7a8c56fdc1b..68450db57d3 100644 --- a/gramps/gen/db/upgrade.py +++ b/gramps/gen/db/upgrade.py @@ -3,6 +3,7 @@ # # Copyright (C) 2020-2016 Gramps Development Team # Copyright (C) 2020 Paul Culley +# Copyright (C) 2024 Doug Blank # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -35,6 +36,8 @@ # # ------------------------------------------------------------------------ from gramps.cli.clidbman import NAME_FILE +from gramps.gen.db.dbconst import CLASS_TO_KEY_MAP +from gramps.gen.lib.serialize import to_dict from gramps.gen.lib import EventType, NameOriginType, Tag, MarkerType from gramps.gen.utils.file import create_checksum from gramps.gen.utils.id import create_id @@ -52,12 +55,66 @@ TAG_KEY, ) from ..const import GRAMPS_LOCALE as glocale +from .conversion_tools import convert_21 +from gramps.gen.lib.serialize import to_dict _ = glocale.translation.gettext LOG = logging.getLogger(".upgrade") +def gramps_upgrade_21(self): + """ + Add json_data field to tables. + """ + length = 0 + for key in self._get_table_func(): + count_func = self._get_table_func(key, "count_func") + length += count_func() + + self.set_total(length) + + # First, do metadata: + + self._txn_begin() + self.set_serializer("blob") + self.upgrade_table_for_json_data("metadata") + keys = self._get_metadata_keys() + for key in keys: + self.set_serializer("blob") + value = self._get_metadata(key, "not-found") + if value != "not-found": + # Save to json_data in current format + self.set_serializer("json") + self._set_metadata(key, value, use_txn=False) + + for table_name in self._get_table_func(): + # For each table, alter the database in an appropriate way: + self.upgrade_table_for_json_data(table_name.lower()) + + get_array_from_handle = self._get_table_func(table_name, "raw_func") + get_object_from_handle = self._get_table_func(table_name, "handle_func") + get_handles = self._get_table_func(table_name, "handles_func") + commit_func = self._get_table_func(table_name, "commit_func") + key = CLASS_TO_KEY_MAP[table_name] + for handle in get_handles(): + # Load from blob: + self.set_serializer("blob") + array = get_array_from_handle(handle) + # Save to json_data in version 21 format + json_data = convert_21(table_name, array) + self.set_serializer("json") + # We can use commit_raw as long as json_data + # has "handle", "_class", and uses json.dumps() + self._commit_raw(json_data, key) + self.update() + + self.set_serializer("json") + self._set_metadata("version", 21, use_txn=False) + self._txn_commit() + # Bump up database version. Separate transaction to save metadata. + + def gramps_upgrade_20(self): """ Placeholder update. diff --git a/gramps/gen/display/name.py b/gramps/gen/display/name.py index 7239c1f315c..1dcf544d300 100644 --- a/gramps/gen/display/name.py +++ b/gramps/gen/display/name.py @@ -4,6 +4,7 @@ # Copyright (C) 2004-2007 Donald N. Allingham # Copyright (C) 2010 Brian G. Matherly # Copyright (C) 2014 Paul Franklin +# Copyright (C) 2024 Doug Blank # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -72,6 +73,7 @@ _ = glocale.translation.sgettext from ..lib.name import Name from ..lib.nameorigintype import NameOriginType +from ..lib.serialize import to_dict try: from ..config import config @@ -86,25 +88,8 @@ # Constants # # ------------------------------------------------------------------------- -_FIRSTNAME = 4 -_SURNAME_LIST = 5 -_SUFFIX = 6 -_TITLE = 7 -_TYPE = 8 -_GROUP = 9 -_SORT = 10 -_DISPLAY = 11 -_CALL = 12 -_NICK = 13 -_FAMNICK = 14 -_SURNAME_IN_LIST = 0 -_PREFIX_IN_LIST = 1 -_PRIMARY_IN_LIST = 2 -_TYPE_IN_LIST = 3 -_CONNECTOR_IN_LIST = 4 _ORIGINPATRO = NameOriginType.PATRONYMIC _ORIGINMATRO = NameOriginType.MATRONYMIC - _ACT = True _INA = False @@ -170,7 +155,7 @@ def _raw_primary_surname(raw_surn_data_list): global PAT_AS_SURN nrsur = len(raw_surn_data_list) for raw_surn_data in raw_surn_data_list: - if raw_surn_data[_PRIMARY_IN_LIST]: + if raw_surn_data["primary"]: # if there are multiple surnames, return the primary. If there # is only one surname, then primary has little meaning, and we # assume a pa/matronymic should not be given as primary as it @@ -179,8 +164,8 @@ def _raw_primary_surname(raw_surn_data_list): not PAT_AS_SURN and nrsur == 1 and ( - raw_surn_data[_TYPE_IN_LIST][0] == _ORIGINPATRO - or raw_surn_data[_TYPE_IN_LIST][0] == _ORIGINMATRO + raw_surn_data["origintype"]["string"] == _ORIGINPATRO + or raw_surn_data["origintype"]["string"] == _ORIGINMATRO ) ): return "" @@ -194,18 +179,18 @@ def _raw_primary_surname_only(raw_surn_data_list): global PAT_AS_SURN nrsur = len(raw_surn_data_list) for raw_surn_data in raw_surn_data_list: - if raw_surn_data[_PRIMARY_IN_LIST]: + if raw_surn_data["primary"]: if ( not PAT_AS_SURN and nrsur == 1 and ( - raw_surn_data[_TYPE_IN_LIST][0] == _ORIGINPATRO - or raw_surn_data[_TYPE_IN_LIST][0] == _ORIGINMATRO + raw_surn_data["origintype"]["string"] == _ORIGINPATRO + or raw_surn_data["origintype"]["string"] == _ORIGINMATRO ) ): return "" else: - return raw_surn_data[_SURNAME_IN_LIST] + return raw_surn_data["surname"] return "" @@ -214,18 +199,18 @@ def _raw_primary_prefix_only(raw_surn_data_list): global PAT_AS_SURN nrsur = len(raw_surn_data_list) for raw_surn_data in raw_surn_data_list: - if raw_surn_data[_PRIMARY_IN_LIST]: + if raw_surn_data["primary"]: if ( not PAT_AS_SURN and nrsur == 1 and ( - raw_surn_data[_TYPE_IN_LIST][0] == _ORIGINPATRO - or raw_surn_data[_TYPE_IN_LIST][0] == _ORIGINMATRO + raw_surn_data["origintype"]["string"] == _ORIGINPATRO + or raw_surn_data["origintype"]["string"] == _ORIGINMATRO ) ): return "" else: - return raw_surn_data[_PREFIX_IN_LIST] + return raw_surn_data["prefix"] return "" @@ -234,18 +219,18 @@ def _raw_primary_conn_only(raw_surn_data_list): global PAT_AS_SURN nrsur = len(raw_surn_data_list) for raw_surn_data in raw_surn_data_list: - if raw_surn_data[_PRIMARY_IN_LIST]: + if raw_surn_data["primary"]: if ( not PAT_AS_SURN and nrsur == 1 and ( - raw_surn_data[_TYPE_IN_LIST][0] == _ORIGINPATRO - or raw_surn_data[_TYPE_IN_LIST][0] == _ORIGINMATRO + raw_surn_data["origintype"]["string"] == _ORIGINPATRO + or raw_surn_data["origintype"]["string"] == _ORIGINMATRO ) ): return "" else: - return raw_surn_data[_CONNECTOR_IN_LIST] + return raw_surn_data["connector"] return "" @@ -253,8 +238,8 @@ def _raw_patro_surname(raw_surn_data_list): """method for the 'y' symbol: patronymic surname""" for raw_surn_data in raw_surn_data_list: if ( - raw_surn_data[_TYPE_IN_LIST][0] == _ORIGINPATRO - or raw_surn_data[_TYPE_IN_LIST][0] == _ORIGINMATRO + raw_surn_data["origintype"]["string"] == _ORIGINPATRO + or raw_surn_data["origintype"]["string"] == _ORIGINMATRO ): return __format_raw_surname(raw_surn_data).strip() return "" @@ -264,10 +249,10 @@ def _raw_patro_surname_only(raw_surn_data_list): """method for the '1y' symbol: patronymic surname only""" for raw_surn_data in raw_surn_data_list: if ( - raw_surn_data[_TYPE_IN_LIST][0] == _ORIGINPATRO - or raw_surn_data[_TYPE_IN_LIST][0] == _ORIGINMATRO + raw_surn_data["origintype"]["string"] == _ORIGINPATRO + or raw_surn_data["origintype"]["string"] == _ORIGINMATRO ): - result = "%s" % (raw_surn_data[_SURNAME_IN_LIST]) + result = "%s" % (raw_surn_data["surname"]) return " ".join(result.split()) return "" @@ -276,10 +261,10 @@ def _raw_patro_prefix_only(raw_surn_data_list): """method for the '0y' symbol: patronymic prefix only""" for raw_surn_data in raw_surn_data_list: if ( - raw_surn_data[_TYPE_IN_LIST][0] == _ORIGINPATRO - or raw_surn_data[_TYPE_IN_LIST][0] == _ORIGINMATRO + raw_surn_data["origintype"]["string"] == _ORIGINPATRO + or raw_surn_data["origintype"]["string"] == _ORIGINMATRO ): - result = "%s" % (raw_surn_data[_PREFIX_IN_LIST]) + result = "%s" % (raw_surn_data["prefix"]) return " ".join(result.split()) return "" @@ -288,10 +273,10 @@ def _raw_patro_conn_only(raw_surn_data_list): """method for the '2y' symbol: patronymic conn only""" for raw_surn_data in raw_surn_data_list: if ( - raw_surn_data[_TYPE_IN_LIST][0] == _ORIGINPATRO - or raw_surn_data[_TYPE_IN_LIST][0] == _ORIGINMATRO + raw_surn_data["origintype"]["string"] == _ORIGINPATRO + or raw_surn_data["origintype"]["string"] == _ORIGINMATRO ): - result = "%s" % (raw_surn_data[_CONNECTOR_IN_LIST]) + result = "%s" % (raw_surn_data["connector"]) return " ".join(result.split()) return "" @@ -303,9 +288,9 @@ def _raw_nonpatro_surname(raw_surn_data_list): result = "" for raw_surn_data in raw_surn_data_list: if ( - (not raw_surn_data[_PRIMARY_IN_LIST]) - and raw_surn_data[_TYPE_IN_LIST][0] != _ORIGINPATRO - and raw_surn_data[_TYPE_IN_LIST][0] != _ORIGINMATRO + (not raw_surn_data["primary"]) + and raw_surn_data["origintype"]["string"] != _ORIGINPATRO + and raw_surn_data["origintype"]["string"] != _ORIGINMATRO ): result += __format_raw_surname(raw_surn_data) return result.strip() @@ -315,7 +300,7 @@ def _raw_nonprimary_surname(raw_surn_data_list): """method for the 'r' symbol: nonprimary surnames""" result = "" for raw_surn_data in raw_surn_data_list: - if not raw_surn_data[_PRIMARY_IN_LIST]: + if not raw_surn_data["primary"]: result += __format_raw_surname(raw_surn_data) return result.strip() @@ -324,7 +309,7 @@ def _raw_prefix_surname(raw_surn_data_list): """method for the 'p' symbol: all prefixes""" result = "" for raw_surn_data in raw_surn_data_list: - result += "%s " % (raw_surn_data[_PREFIX_IN_LIST]) + result += "%s " % (raw_surn_data["prefix"]) return " ".join(result.split()).strip() @@ -332,7 +317,7 @@ def _raw_single_surname(raw_surn_data_list): """method for the 'q' symbol: surnames without prefix and connectors""" result = "" for raw_surn_data in raw_surn_data_list: - result += "%s " % (raw_surn_data[_SURNAME_IN_LIST]) + result += "%s " % (raw_surn_data["surname"]) return " ".join(result.split()).strip() @@ -359,14 +344,14 @@ def __format_raw_surname(raw_surn_data): If the connector is a hyphen, don't pad it with spaces. """ - result = raw_surn_data[_PREFIX_IN_LIST] + result = raw_surn_data["prefix"] if result: result += " " - result += raw_surn_data[_SURNAME_IN_LIST] - if result and raw_surn_data[_CONNECTOR_IN_LIST] != "-": - result += " %s " % raw_surn_data[_CONNECTOR_IN_LIST] + result += raw_surn_data["surname"] + if result and raw_surn_data["connector"] != "-": + result += " %s " % raw_surn_data["connector"] else: - result += raw_surn_data[_CONNECTOR_IN_LIST] + result += raw_surn_data["connector"] return result @@ -460,22 +445,22 @@ def _format_raw_fn(self, fmt_str): def _raw_lnfn(self, raw_data): result = self.LNFN_STR % ( - _raw_full_surname(raw_data[_SURNAME_LIST]), - raw_data[_FIRSTNAME], - raw_data[_SUFFIX], + _raw_full_surname(raw_data["surname_list"]), + raw_data["first_name"], + raw_data["suffix"], ) return " ".join(result.split()) def _raw_fnln(self, raw_data): result = "%s %s %s" % ( - raw_data[_FIRSTNAME], - _raw_full_surname(raw_data[_SURNAME_LIST]), - raw_data[_SUFFIX], + raw_data["first_name"], + _raw_full_surname(raw_data["surname_list"]), + raw_data["suffix"], ) return " ".join(result.split()) def _raw_fn(self, raw_data): - result = raw_data[_FIRSTNAME] + result = raw_data["first_name"] return " ".join(result.split()) def clear_custom_formats(self): @@ -616,9 +601,9 @@ def _gen_raw_func(self, format_str): The new function is of the form:: def fn(raw_data): - return "%s %s %s" % (raw_data[_TITLE], - raw_data[_FIRSTNAME], - raw_data[_SUFFIX]) + return "%s %s %s" % (raw_data["title"], + raw_data["first_name"], + raw_data["suffix"]) Specific symbols for parts of a name are defined (keywords given): 't' : title = title @@ -650,88 +635,88 @@ def fn(raw_data): # called to fill in each format flag. # Dictionary is "code": ("expression", "keyword", "i18n-keyword") d = { - "t": ("raw_data[_TITLE]", "title", _("title", "Person")), - "f": ("raw_data[_FIRSTNAME]", "given", _("given")), + "t": ("raw_data['title']", "title", _("title", "Person")), + "f": ("raw_data['first_name']", "given", _("given")), "l": ( - "_raw_full_surname(raw_data[_SURNAME_LIST])", + "_raw_full_surname(raw_data['surname_list'])", "surname", _("surname"), ), - "s": ("raw_data[_SUFFIX]", "suffix", _("suffix")), - "c": ("raw_data[_CALL]", "call", _("call", "Name")), + "s": ("raw_data['suffix']", "suffix", _("suffix")), + "c": ("raw_data['call']", "call", _("call", "Name")), "x": ( - "(raw_data[_NICK] or raw_data[_CALL] or raw_data[_FIRSTNAME].split(' ')[0])", + "(raw_data['nick'] or raw_data['call'] or raw_data['first_name'].split(' ')[0])", "common", _("common", "Name"), ), "i": ( "''.join([word[0] +'.' for word in ('. ' +" - + " raw_data[_FIRSTNAME]).split()][1:])", + + " raw_data['first_name']).split()][1:])", "initials", _("initials"), ), "m": ( - "_raw_primary_surname(raw_data[_SURNAME_LIST])", + "_raw_primary_surname(raw_data['surname_list'])", "primary", _("primary", "Name"), ), "0m": ( - "_raw_primary_prefix_only(raw_data[_SURNAME_LIST])", + "_raw_primary_prefix_only(raw_data['surname_list'])", "primary[pre]", _("primary[pre]"), ), "1m": ( - "_raw_primary_surname_only(raw_data[_SURNAME_LIST])", + "_raw_primary_surname_only(raw_data['surname_list'])", "primary[sur]", _("primary[sur]"), ), "2m": ( - "_raw_primary_conn_only(raw_data[_SURNAME_LIST])", + "_raw_primary_conn_only(raw_data['surname_list'])", "primary[con]", _("primary[con]"), ), "y": ( - "_raw_patro_surname(raw_data[_SURNAME_LIST])", + "_raw_patro_surname(raw_data['surname_list'])", "patronymic", _("patronymic"), ), "0y": ( - "_raw_patro_prefix_only(raw_data[_SURNAME_LIST])", + "_raw_patro_prefix_only(raw_data['surname_list'])", "patronymic[pre]", _("patronymic[pre]"), ), "1y": ( - "_raw_patro_surname_only(raw_data[_SURNAME_LIST])", + "_raw_patro_surname_only(raw_data['surname_list'])", "patronymic[sur]", _("patronymic[sur]"), ), "2y": ( - "_raw_patro_conn_only(raw_data[_SURNAME_LIST])", + "_raw_patro_conn_only(raw_data['surname_list'])", "patronymic[con]", _("patronymic[con]"), ), "o": ( - "_raw_nonpatro_surname(raw_data[_SURNAME_LIST])", + "_raw_nonpatro_surname(raw_data['surname_list'])", "notpatronymic", _("notpatronymic"), ), "r": ( - "_raw_nonprimary_surname(raw_data[_SURNAME_LIST])", + "_raw_nonprimary_surname(raw_data['surname_list'])", "rest", _("rest", "Remaining names"), ), "p": ( - "_raw_prefix_surname(raw_data[_SURNAME_LIST])", + "_raw_prefix_surname(raw_data['surname_list'])", "prefix", _("prefix"), ), "q": ( - "_raw_single_surname(raw_data[_SURNAME_LIST])", + "_raw_single_surname(raw_data['surname_list'])", "rawsurnames", _("rawsurnames"), ), - "n": ("raw_data[_NICK]", "nickname", _("nickname")), - "g": ("raw_data[_FAMNICK]", "familynick", _("familynick")), + "n": ("raw_data['nick']", "nickname", _("nickname")), + "g": ("raw_data['famnick']", "familynick", _("familynick")), } args = "raw_data" return self._make_fn(format_str, d, args) @@ -925,7 +910,7 @@ def _format_str_base( try: s = func( first, - [surn.serialize() for surn in surname_list], + [to_dict(surn) for surn in surname_list], suffix, title, call, @@ -1016,7 +1001,7 @@ def raw_sorted_name(self, raw_data): :returns: Returns the :class:`~.name.Name` string representation :rtype: str """ - num = self._is_format_valid(raw_data[_SORT]) + num = self._is_format_valid(raw_data["sort_as"]) return self.name_formats[num][_F_RAWFN](raw_data) def display(self, person): @@ -1096,7 +1081,7 @@ def raw_display_name(self, raw_data): :returns: Returns the :class:`~.name.Name` string representation :rtype: str """ - num = self._is_format_valid(raw_data[_DISPLAY]) + num = self._is_format_valid(raw_data["display_as"]) return self.name_formats[num][_F_RAWFN](raw_data) def display_given(self, person): @@ -1145,18 +1130,18 @@ def name_grouping_data(self, db, pn): :returns: Returns the groupname string representation :rtype: str """ - if pn[_GROUP]: - return pn[_GROUP] - name = pn[_GROUP] + if pn["group_as"]: + return pn["group_as"] + name = pn["group_as"] if not name: # if we have no primary surname, perhaps we have a # patronymic/matronynic name ? - srnme = pn[_ORIGINPATRO] + srnme = pn["surname_list"] surname = [] for _surname in srnme: if ( - _surname[_TYPE_IN_LIST][0] == _ORIGINPATRO - or _surname[_TYPE_IN_LIST][0] == _ORIGINMATRO + _surname["origintype"]["string"] == _ORIGINPATRO + or _surname["origintype"]["string"] == _ORIGINMATRO ): # Yes, we have one. surname = [_surname] @@ -1166,7 +1151,7 @@ def name_grouping_data(self, db, pn): name = db.get_name_group_mapping(name1) if not name: name = db.get_name_group_mapping( - _raw_primary_surname_only(pn[_SURNAME_LIST]) + _raw_primary_surname_only(pn["surname_list"]) ) return name diff --git a/gramps/gen/filters/_genericfilter.py b/gramps/gen/filters/_genericfilter.py index 87fd849f56b..02dcfd2b82b 100644 --- a/gramps/gen/filters/_genericfilter.py +++ b/gramps/gen/filters/_genericfilter.py @@ -3,7 +3,7 @@ # # Copyright (C) 2002-2006 Donald N. Allingham # Copyright (C) 2011 Tim G L Lyons -# Copyright (C) 2012 Doug Blank +# Copyright (C) 2012,2024 Doug Blank # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -39,6 +39,7 @@ from ..lib.media import Media from ..lib.note import Note from ..lib.tag import Tag +from ..lib.serialize import from_dict from ..const import GRAMPS_LOCALE as glocale _ = glocale.translation.gettext @@ -145,8 +146,7 @@ def check_func(self, db, id_list, task, user=None, tupleind=None, tree=False): if id_list is None: with self.get_tree_cursor(db) if tree else self.get_cursor(db) as cursor: for handle, data in cursor: - person = self.make_obj() - person.unserialize(data) + person = from_dict(data) if user: user.step_progress() if task(db, person) != self.invert: @@ -174,8 +174,7 @@ def check_and(self, db, id_list, user=None, tupleind=None, tree=False): if id_list is None: with self.get_tree_cursor(db) if tree else self.get_cursor(db) as cursor: for handle, data in cursor: - person = self.make_obj() - person.unserialize(data) + person = from_dict(data) if user: user.step_progress() val = all(rule.apply(db, person) for rule in flist) diff --git a/gramps/gen/lib/date.py b/gramps/gen/lib/date.py index 81d942174fb..46946f780ff 100644 --- a/gramps/gen/lib/date.py +++ b/gramps/gen/lib/date.py @@ -1,11 +1,11 @@ # # Gramps - a GTK+/GNOME based genealogy program # -# Copyright (C) 2000-2007 Donald N. Allingham -# Copyright (C) 2009-2013 Douglas S. Blank -# Copyright (C) 2013 Paul Franklin -# Copyright (C) 2013-2014 Vassilii Khachaturov -# Copyright (C) 2017,2024 Nick Hall +# Copyright (C) 2000-2007 Donald N. Allingham +# Copyright (C) 2009-2013,2024 Douglas S. Blank +# Copyright (C) 2013 Paul Franklin +# Copyright (C) 2013-2014 Vassilii Khachaturov +# Copyright (C) 2017,2024 Nick Hall # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -602,6 +602,7 @@ class Date(BaseObject): EMPTY = (0, 0, 0, False) + # These are positions in JSON dateval: _POS_DAY = 0 _POS_MON = 1 _POS_YR = 2 diff --git a/gramps/gen/lib/grampstype.py b/gramps/gen/lib/grampstype.py index 39818ea8774..a5021fe7003 100644 --- a/gramps/gen/lib/grampstype.py +++ b/gramps/gen/lib/grampstype.py @@ -3,6 +3,7 @@ # # Copyright (C) 2000-2007 Donald N. Allingham # Copyright (C) 2017,2024 Nick Hall +# Copyright (C) 2024 Doug Blank # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -87,14 +88,6 @@ class GrampsType(metaclass=GrampsTypeMeta): List of indices to ignore (obsolete/retired entries). (gramps policy is never to delete type values, or reuse the name (TOKEN) of any specific type value) - :cvar POS_: (int) - Position of attribute in the serialized format of - an instance. - - .. warning:: The POS_ class variables reflect the serialized object, - they have to be updated in case the data structure or the - :meth:`serialize` method changes! - :cvar _CUSTOM: (int) a custom type object :cvar _DEFAULT: (int) the default type, used on creation @@ -102,8 +95,6 @@ class GrampsType(metaclass=GrampsTypeMeta): :attribute string: (str) Returns or sets string value """ - (POS_VALUE, POS_STRING) = list(range(2)) - _CUSTOM = 0 _DEFAULT = 0 diff --git a/gramps/gen/lib/locationbase.py b/gramps/gen/lib/locationbase.py index 56daf4060ea..6a2ec5798ba 100644 --- a/gramps/gen/lib/locationbase.py +++ b/gramps/gen/lib/locationbase.py @@ -189,3 +189,44 @@ def set_county(self, data): def get_county(self): """Return the county name of the LocationBase object.""" return self.county + + def get_object_state(self): + """ + Get the current object state as a dictionary. + + By default this returns the public attributes of the instance. This + method can be overridden if the class requires other attributes or + properties to be saved. + + This method is called to provide the information required to serialize + the object. + + :returns: Returns a dictionary of attributes that represent the state + of the object. + :rtype: dict + """ + attr_dict = dict( + (key, value) + for key, value in self.__dict__.items() + if not key.startswith("_") + ) + attr_dict["_class"] = self.__class__.__name__ + return attr_dict + + def set_object_state(self, attr_dict): + """ + Set the current object state using information provided in the given + dictionary. + + By default this sets the state of the object assuming that all items in + the dictionary map to public attributes. This method can be overridden + to set the state using custom functionality. For performance reasons + it is useful to set a property without calling its setter function. As + JSON provides no distinction between tuples and lists, this method can + also be use to convert lists into tuples where required. + + :param attr_dict: A dictionary of attributes that represent the state of + the object. + :type attr_dict: dict + """ + self.__dict__.update(attr_dict) diff --git a/gramps/gen/lib/note.py b/gramps/gen/lib/note.py index 4fffedd3c45..d19a1fd9591 100644 --- a/gramps/gen/lib/note.py +++ b/gramps/gen/lib/note.py @@ -4,6 +4,7 @@ # Copyright (C) 2000-2007 Donald N. Allingham # Copyright (C) 2010 Michiel D. Nauta # Copyright (C) 2010,2017 Nick Hall +# Copyright (C) 2024 Doug Blank # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -62,27 +63,10 @@ class Note(BasicPrimaryObject): :cvar FLOWED: indicates flowed format :cvar FORMATTED: indicates formatted format (respecting whitespace needed) - :cvar POS_: (int) Position of attribute in the serialized format of - an instance. - - .. warning:: The POS_ class variables reflect the serialized object, - they have to be updated in case the data structure or the - :meth:`serialize` method changes! """ (FLOWED, FORMATTED) = list(range(2)) - ( - POS_HANDLE, - POS_ID, - POS_TEXT, - POS_FORMAT, - POS_TYPE, - POS_CHANGE, - POS_TAGS, - POS_PRIVATE, - ) = list(range(8)) - def __init__(self, text=""): """Create a new Note object, initializing from the passed string.""" BasicPrimaryObject.__init__(self) diff --git a/gramps/gen/lib/serialize.py b/gramps/gen/lib/serialize.py index 2a1d863bf1e..40ebe1568b9 100644 --- a/gramps/gen/lib/serialize.py +++ b/gramps/gen/lib/serialize.py @@ -2,6 +2,7 @@ # Gramps - a GTK+/GNOME based genealogy program # # Copyright (C) 2017,2024 Nick Hall +# Copyright (C) 2024 Doug Blank # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -28,6 +29,8 @@ # # ------------------------------------------------------------------------ import json +import pickle +import logging # ------------------------------------------------------------------------ # @@ -36,6 +39,8 @@ # ------------------------------------------------------------------------ import gramps.gen.lib as lib +LOG = logging.getLogger(".serialize") + def __object_hook(obj_dict): _class = obj_dict.pop("_class") @@ -71,3 +76,147 @@ def from_json(data): :rtype: object """ return json.loads(data, object_hook=__object_hook) + + +def to_dict(obj): + """ + Convert a Gramps object into a struct. + + :param obj: The object to be serialized. + :type obj: object + :returns: A dictionary. + :rtype: dict + """ + return json.loads(to_json(obj)) + + +def from_dict(dict): + """ + Convert a dictionary into a Gramps object. + + :param dict: The dictionary to be unserialized. + :type dict: dict + :returns: A Gramps object. + :rtype: object + """ + return from_json(json.dumps(dict)) + + +class BlobSerializer: + """ + Serializer for blob data + + In this serializer, data is a nested array, + and string is pickled bytes. + """ + + data_field = "blob_data" + metadata_field = "value" + + @staticmethod + def data_to_object(obj_class, data): + LOG.debug("blob, data_to_object: %s(%r)", obj_class, data[0] if data else data) + return obj_class.create(data) + + @staticmethod + def string_to_object(obj_class, bytes): + LOG.debug("blob, string_to_object: %r...", bytes[:35]) + return obj_class.create(pickle.loads(bytes)) + + @staticmethod + def string_to_data(bytes): + LOG.debug("blob, string_to_object: %r...", bytes[:35]) + return pickle.loads(bytes) + + @staticmethod + def object_to_string(obj): + LOG.debug("blob, object_to_string: %s...", obj) + return pickle.dumps(obj.serialize()) + + @staticmethod + def data_to_string(data): + LOG.debug("blob, data_to_string: %s...", data[0] if data else data) + return pickle.dumps(data) + + @staticmethod + def metadata_to_object(blob): + return pickle.loads(blob) + + @staticmethod + def object_to_metadata(value): + return pickle.dumps(value) + + +class JSONSerializer: + """ + Serializer for JSON data. + + In this serializer, data is a dict, + and string is a JSON string. + """ + + data_field = "json_data" + metadata_field = "json_data" + + @staticmethod + def data_to_object(obj_class, data): + LOG.debug( + "json, data_to_object: {'_class': %r, ...}", + data["_class"] if (data and "_class" in data) else data, + ) + return from_dict(data) + + @staticmethod + def string_to_object(obj_class, string): + LOG.debug("json, string_to_object: %r...", string[:65]) + return from_json(string) + + @staticmethod + def string_to_data(string): + LOG.debug("json, string_to_data: %r...", string[:65]) + return json.loads(string) + + @staticmethod + def object_to_string(obj): + LOG.debug("json, object_to_string: %s...", obj) + return to_json(obj) + + @staticmethod + def data_to_string(data): + LOG.debug( + "json, data_to_string: {'_class': %r, ...}", + data["_class"] if (data and "_class" in data) else data, + ) + return json.dumps(data) + + @staticmethod + def metadata_to_object(string): + doc = json.loads(string) + type_name = doc["type"] + if type_name in ("int", "str", "list"): + return doc["value"] + elif type_name == "set": + return set(doc["value"]) + elif type_name == "tuple": + return tuple(doc["value"]) + elif type_name == "dict": + return doc["value"] + elif type_name == "Researcher": + return from_dict(doc["value"]) + else: + return doc["value"] + + @staticmethod + def object_to_metadata(value): + type_name = type(value).__name__ + if type_name in ("set", "tuple"): + value = list(value) + elif type_name == "Researcher": + value = to_dict(value) + elif type_name not in ("int", "str", "list"): + value = json.loads(to_json(value)) + data = { + "type": type_name, + "value": value, + } + return json.dumps(data) diff --git a/gramps/gen/lib/styledtext.py b/gramps/gen/lib/styledtext.py index 8f358eac0d5..47d4eed4b4e 100644 --- a/gramps/gen/lib/styledtext.py +++ b/gramps/gen/lib/styledtext.py @@ -2,7 +2,7 @@ # Gramps - a GTK+/GNOME based genealogy program # # Copyright (C) 2008 Zsolt Foldvari -# Copyright (C) 2013 Doug Blank +# Copyright (C) 2013,2024 Doug Blank # Copyright (C) 2017,2024 Nick Hall # # This program is free software; you can redistribute it and/or modify @@ -68,15 +68,6 @@ class StyledText(BaseObject): :ivar tags: (list of :py:class:`.StyledTextTag`) Text tags holding formatting information for the string. - :cvar POS_TEXT: Position of *string* attribute in the serialized format of - an instance. - :cvar POS_TAGS: (int) Position of *tags* attribute in the serialized format - of an instance. - - .. warning:: The POS_ class variables reflect the serialized object, - they have to be updated in case the data structure or the - :py:meth:`serialize` method changes! - .. note:: 1. There is no sanity check of tags in :py:meth:`__init__`, because when a :py:class:`StyledText` is displayed it is passed to a @@ -91,8 +82,6 @@ class StyledText(BaseObject): so if you intend to use a source tag more than once, copy it for use. """ - (POS_TEXT, POS_TAGS) = list(range(2)) - def __init__(self, text="", tags=None): """Setup initial instance variable values.""" self._string = text diff --git a/gramps/gen/lib/surname.py b/gramps/gen/lib/surname.py index 84aff9e3e2d..577f9a472ec 100644 --- a/gramps/gen/lib/surname.py +++ b/gramps/gen/lib/surname.py @@ -69,6 +69,25 @@ def __init__(self, source=None, data=None): if data: self.unserialize(data) + def get_object_state(self): + """ + Get the current object state as a dictionary. + """ + attr_dict = dict( + (key, value) + for key, value in self.__dict__.items() + if not key.startswith("_") + ) + attr_dict["_class"] = self.__class__.__name__ + return attr_dict + + def set_object_state(self, attr_dict): + """ + Set the current object state using information provided in the given + dictionary. + """ + self.__dict__.update(attr_dict) + def serialize(self): """ Convert the object to a serialized tuple of data. diff --git a/gramps/gen/lib/test/serialize_test.py b/gramps/gen/lib/test/serialize_test.py index bbc44e8324f..5e4a0b43b04 100644 --- a/gramps/gen/lib/test/serialize_test.py +++ b/gramps/gen/lib/test/serialize_test.py @@ -129,7 +129,7 @@ def test(self): setattr(DatabaseCheck, name, test) #### # def test2(self): - # self.assertEqual(obj.serialize(), from_struct(struct).serialize()) + # self.assertEqual(obj.serialize(), from_dict(struct).serialize()) # name = "test_create_%s_%s" % (obj.__class__.__name__, obj.handle) # setattr(DatabaseCheck, name, test2) diff --git a/gramps/gen/merge/diff.py b/gramps/gen/merge/diff.py index 2950470cff1..b6243a07017 100644 --- a/gramps/gen/merge/diff.py +++ b/gramps/gen/merge/diff.py @@ -1,7 +1,7 @@ # # Gramps - a GTK+/GNOME based genealogy program # -# Copyright (C) 2012 Doug Blank +# Copyright (C) 2012,2024 Doug Blank # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -25,19 +25,12 @@ import json from ..db.utils import import_as_dict -from ..lib.serialize import to_json +from ..lib.serialize import to_dict, from_dict from ..const import GRAMPS_LOCALE as glocale _ = glocale.translation.gettext -def to_struct(obj): - """ - Convert an object into a struct. - """ - return json.loads(to_json(obj)) - - def diff_dates(json1, json2): """ Compare two json date objects. Returns True if different. @@ -136,7 +129,7 @@ def diff_dbs(db1, db2, user): if handles1[p1] == handles2[p2]: # in both item1 = handle_func1(handles1[p1]) item2 = handle_func2(handles2[p2]) - diff = diff_items(item, to_struct(item1), to_struct(item2)) + diff = diff_items(item, to_dict(item1), to_dict(item2)) if diff: diffs += [(item, item1, item2)] # else same! diff --git a/gramps/gui/editors/editsecondary.py b/gramps/gui/editors/editsecondary.py index 06dd12677d3..7e15e03925d 100644 --- a/gramps/gui/editors/editsecondary.py +++ b/gramps/gui/editors/editsecondary.py @@ -2,7 +2,8 @@ # Gramps - a GTK+/GNOME based genealogy program # # Copyright (C) 2000-2006 Donald N. Allingham -# 2009 Gary Burton +# Copyright (C) 2009 Gary Burton +# Copyright (C) 2024 Doug Blank # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -27,6 +28,7 @@ from ..managedwindow import ManagedWindow from ..display import display_help from gramps.gen.config import config +from gramps.gen.lib.serialize import to_dict, from_dict from ..dbguielement import DbGUIElement @@ -35,7 +37,7 @@ def __init__(self, state, uistate, track, obj, callback=None): """Create an edit window. Associates a person with the window.""" self.obj = obj - self.old_obj = obj.serialize() + self.old_obj = to_dict(obj) self.dbstate = state self.uistate = uistate self.db = state.db @@ -136,7 +138,7 @@ def canceledits(self, *obj): """ Undo the edits that happened on this secondary object """ - self.obj.unserialize(self.old_obj) + self.obj = from_dict(self.old_obj) self.close(obj) def close(self, *obj): diff --git a/gramps/gui/plug/quick/_quicktable.py b/gramps/gui/plug/quick/_quicktable.py index f9a5bd79eec..9b42c74289a 100644 --- a/gramps/gui/plug/quick/_quicktable.py +++ b/gramps/gui/plug/quick/_quicktable.py @@ -1,8 +1,8 @@ # # Gramps - a GTK+/GNOME based genealogy program # -# Copyright (C) 2008 Donald N. Allingham -# Copyright (C) 2009 Douglas S. Blank +# Copyright (C) 2008 Donald N. Allingham +# Copyright (C) 2009,2024 Douglas S. Blank # Copyright (C) 2011 Tim G L Lyons # # This program is free software; you can redistribute it and/or modify @@ -471,7 +471,7 @@ def write(self, document): model.append( row=([count] + list(rowdata) + [col[count] for col in sort_data]) ) - except KeyError as msg: + except (IndexError, KeyError) as msg: print(msg) if sort_data: print( diff --git a/gramps/gui/views/treemodels/citationbasemodel.py b/gramps/gui/views/treemodels/citationbasemodel.py index a307b8af42f..739c770358a 100644 --- a/gramps/gui/views/treemodels/citationbasemodel.py +++ b/gramps/gui/views/treemodels/citationbasemodel.py @@ -3,6 +3,7 @@ # # Copyright (C) 2000-2006 Donald N. Allingham # Copyright (C) 2011 Tim G L Lyons, Nick Hall +# Copyright (C) 2024 Doug Blank # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -44,37 +45,10 @@ _ = glocale.translation.gettext from gramps.gen.datehandler import format_time, get_date, get_date_valid from gramps.gen.lib import Citation +from gramps.gen.lib.serialize import from_dict from gramps.gen.utils.string import conf_strings from gramps.gen.config import config -# ------------------------------------------------------------------------- -# -# COLUMN constants -# -# ------------------------------------------------------------------------- -# These are the column numbers in the serialize/unserialize interfaces in -# the Citation object -COLUMN_HANDLE = 0 -COLUMN_ID = 1 -COLUMN_DATE = 2 -COLUMN_PAGE = 3 -COLUMN_CONFIDENCE = 4 -COLUMN_SOURCE = 5 -COLUMN_CHANGE = 9 -COLUMN_TAGS = 10 -COLUMN_PRIV = 11 - -# Data for the Source object -COLUMN2_HANDLE = 0 -COLUMN2_ID = 1 -COLUMN2_TITLE = 2 -COLUMN2_AUTHOR = 3 -COLUMN2_PUBINFO = 4 -COLUMN2_ABBREV = 7 -COLUMN2_CHANGE = 8 -COLUMN2_TAGS = 11 -COLUMN2_PRIV = 12 - INVALID_DATE_FORMAT = config.get("preferences.invalid-date-format") @@ -87,12 +61,14 @@ class CitationBaseModel: # Fields access when 'data' is a Citation def citation_date(self, data): - if data[COLUMN_DATE]: - citation = Citation() - citation.unserialize(data) + if data["date"]: + citation = from_dict(data) date_str = get_date(citation) if date_str != "": retval = escape(date_str) + else: + retval = "" + if not get_date_valid(citation): return INVALID_DATE_FORMAT % retval else: @@ -100,9 +76,8 @@ def citation_date(self, data): return "" def citation_sort_date(self, data): - if data[COLUMN_DATE]: - citation = Citation() - citation.unserialize(data) + if data["date"]: + citation = from_dict(data) retval = "%09d" % citation.get_date_object().get_sort_value() if not get_date_valid(citation): return INVALID_DATE_FORMAT % retval @@ -111,21 +86,21 @@ def citation_sort_date(self, data): return "" def citation_id(self, data): - return data[COLUMN_ID] + return data["gramps_id"] def citation_page(self, data): - return data[COLUMN_PAGE] + return data["page"] def citation_sort_confidence(self, data): - if data[COLUMN_CONFIDENCE]: - return str(data[COLUMN_CONFIDENCE]) + if data["confidence"]: + return str(data["confidence"]) return "" def citation_confidence(self, data): - return _(conf_strings[data[COLUMN_CONFIDENCE]]) + return _(conf_strings[data["confidence"]]) def citation_private(self, data): - if data[COLUMN_PRIV]: + if data["private"]: return "gramps-lock" else: # There is a problem returning None here. @@ -135,7 +110,7 @@ def citation_tags(self, data): """ Return the sorted list of tags. """ - tag_list = list(map(self.get_tag_name, data[COLUMN_TAGS])) + tag_list = list(map(self.get_tag_name, data["tag_list"])) # TODO for Arabic, should the next line's comma be translated? return ", ".join(sorted(tag_list, key=glocale.sort_key)) @@ -143,12 +118,12 @@ def citation_tag_color(self, data): """ Return the tag color. """ - tag_handle = data[0] + tag_handle = data["handle"] cached, tag_color = self.get_cached_value(tag_handle, "TAG_COLOR") if not cached: tag_color = "" tag_priority = None - for handle in data[COLUMN_TAGS]: + for handle in data["tag_list"]: tag = self.db.get_tag_from_handle(handle) this_priority = tag.get_priority() if tag_priority is None or this_priority < tag_priority: @@ -158,16 +133,16 @@ def citation_tag_color(self, data): return tag_color def citation_change(self, data): - return format_time(data[COLUMN_CHANGE]) + return format_time(data["change"]) def citation_sort_change(self, data): - return "%012x" % data[COLUMN_CHANGE] + return "%012x" % data["change"] def citation_source(self, data): - return data[COLUMN_SOURCE] + return data["source_handle"] def citation_src_title(self, data): - source_handle = data[COLUMN_SOURCE] + source_handle = data["source_handle"] cached, value = self.get_cached_value(source_handle, "SRC_TITLE") if not cached: try: @@ -179,7 +154,7 @@ def citation_src_title(self, data): return value def citation_src_id(self, data): - source_handle = data[COLUMN_SOURCE] + source_handle = data["source_handle"] cached, value = self.get_cached_value(source_handle, "SRC_ID") if not cached: try: @@ -191,7 +166,7 @@ def citation_src_id(self, data): return value def citation_src_auth(self, data): - source_handle = data[COLUMN_SOURCE] + source_handle = data["source_handle"] cached, value = self.get_cached_value(source_handle, "SRC_AUTH") if not cached: try: @@ -203,7 +178,7 @@ def citation_src_auth(self, data): return value def citation_src_abbr(self, data): - source_handle = data[COLUMN_SOURCE] + source_handle = data["source_handle"] cached, value = self.get_cached_value(source_handle, "SRC_ABBR") if not cached: try: @@ -215,7 +190,7 @@ def citation_src_abbr(self, data): return value def citation_src_pinfo(self, data): - source_handle = data[COLUMN_SOURCE] + source_handle = data["source_handle"] cached, value = self.get_cached_value(source_handle, "SRC_PINFO") if not cached: try: @@ -227,7 +202,7 @@ def citation_src_pinfo(self, data): return value def citation_src_private(self, data): - source_handle = data[COLUMN_SOURCE] + source_handle = data["source_handle"] cached, value = self.get_cached_value(source_handle, "SRC_PRIVATE") if not cached: try: @@ -243,7 +218,7 @@ def citation_src_private(self, data): return value def citation_src_tags(self, data): - source_handle = data[COLUMN_SOURCE] + source_handle = data["source_handle"] cached, value = self.get_cached_value(source_handle, "SRC_TAGS") if not cached: try: @@ -257,7 +232,7 @@ def citation_src_tags(self, data): return value def citation_src_chan(self, data): - source_handle = data[COLUMN_SOURCE] + source_handle = data["source_handle"] cached, value = self.get_cached_value(source_handle, "SRC_CHAN") if not cached: try: @@ -269,7 +244,7 @@ def citation_src_chan(self, data): return value def citation_src_sort_change(self, data): - source_handle = data[COLUMN_SOURCE] + source_handle = data["source_handle"] cached, value = self.get_cached_value(source_handle, "SRC_CHAN") if not cached: try: @@ -283,22 +258,22 @@ def citation_src_sort_change(self, data): # Fields access when 'data' is a Source def source_src_title(self, data): - return data[COLUMN2_TITLE] + return data["title"] def source_src_id(self, data): - return data[COLUMN2_ID] + return data["gramps_id"] def source_src_auth(self, data): - return data[COLUMN2_AUTHOR] + return data["author"] def source_src_abbr(self, data): - return data[COLUMN2_ABBREV] + return data["abbrev"] def source_src_pinfo(self, data): - return data[COLUMN2_PUBINFO] + return data["pubinfo"] def source_src_private(self, data): - if data[COLUMN2_PRIV]: + if data["private"]: return "gramps-lock" else: # There is a problem returning None here. @@ -308,7 +283,7 @@ def source_src_tags(self, data): """ Return the sorted list of tags. """ - tag_list = list(map(self.get_tag_name, data[COLUMN2_TAGS])) + tag_list = list(map(self.get_tag_name, data["tag_list"])) # TODO for Arabic, should the next line's comma be translated? return ", ".join(sorted(tag_list, key=glocale.sort_key)) @@ -316,12 +291,12 @@ def source_src_tag_color(self, data): """ Return the tag color. """ - tag_handle = data[0] + tag_handle = data["handle"] cached, tag_color = self.get_cached_value(tag_handle, "TAG_COLOR") if not cached: tag_color = "" tag_priority = None - for handle in data[COLUMN2_TAGS]: + for handle in data["tag_list"]: tag = self.db.get_tag_from_handle(handle) this_priority = tag.get_priority() if tag_priority is None or this_priority < tag_priority: @@ -331,10 +306,10 @@ def source_src_tag_color(self, data): return tag_color def source_src_chan(self, data): - return format_time(data[COLUMN2_CHANGE]) + return format_time(data["change"]) def source_sort2_change(self, data): - return "%012x" % data[COLUMN2_CHANGE] + return "%012x" % data["change"] def dummy_sort_key(self, data): # dummy sort key for columns that don't have data diff --git a/gramps/gui/views/treemodels/citationtreemodel.py b/gramps/gui/views/treemodels/citationtreemodel.py index 23592fc8102..60ec5b1699d 100644 --- a/gramps/gui/views/treemodels/citationtreemodel.py +++ b/gramps/gui/views/treemodels/citationtreemodel.py @@ -3,6 +3,7 @@ # # Copyright (C) 2000-2006 Donald N. Allingham # Copyright (C) 2011 Tim G L Lyons, Nick Hall +# Copyright (C) 2024 Doug Blank # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -211,17 +212,23 @@ def add_row2(self, handle, data): # add the citation as a child of the source. Otherwise we add the source # first (because citations don't have any meaning without the associated # source) - if self._get_node(data[5]): + if self._get_node(data["source_handle"]): # parent child sortkey handle - self.add_node(data[5], handle, sort_key, handle, secondary=True) + self.add_node( + data["source_handle"], handle, sort_key, handle, secondary=True + ) else: # add the source node first - source_sort_key = self.sort_func(self.map(data[5])) + source_sort_key = self.sort_func(self.map(data["source_handle"])) # parent child sortkey handle - self.add_node(None, data[5], source_sort_key, data[5]) + self.add_node( + None, data["source_handle"], source_sort_key, data["source_handle"] + ) # parent child sortkey handle - self.add_node(data[5], handle, sort_key, handle, secondary=True) + self.add_node( + data["source_handle"], handle, sort_key, handle, secondary=True + ) def on_get_n_columns(self): return len(self.fmap) + 1 diff --git a/gramps/gui/views/treemodels/eventmodel.py b/gramps/gui/views/treemodels/eventmodel.py index 44d9eff7289..e09dd0e4a30 100644 --- a/gramps/gui/views/treemodels/eventmodel.py +++ b/gramps/gui/views/treemodels/eventmodel.py @@ -2,6 +2,7 @@ # Gramps - a GTK+/GNOME based genealogy program # # Copyright (C) 2000-2006 Donald N. Allingham +# Copyright (C) 2024 Doug Blank # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -42,27 +43,13 @@ # ------------------------------------------------------------------------- from gramps.gen.datehandler import format_time, get_date, get_date_valid from gramps.gen.lib import Event, EventType +from gramps.gen.lib.serialize import from_dict from gramps.gen.utils.db import get_participant_from_event from gramps.gen.display.place import displayer as place_displayer from gramps.gen.config import config from .flatbasemodel import FlatBaseModel from gramps.gen.const import GRAMPS_LOCALE as glocale -# ------------------------------------------------------------------------- -# -# Positions in raw data structure -# -# ------------------------------------------------------------------------- -COLUMN_HANDLE = 0 -COLUMN_ID = 1 -COLUMN_TYPE = 2 -COLUMN_DATE = 3 -COLUMN_DESCRIPTION = 4 -COLUMN_PLACE = 5 -COLUMN_CHANGE = 10 -COLUMN_TAGS = 11 -COLUMN_PRIV = 12 - INVALID_DATE_FORMAT = config.get("preferences.invalid-date-format") @@ -134,40 +121,38 @@ def on_get_n_columns(self): return len(self.fmap) + 1 def column_description(self, data): - return data[COLUMN_DESCRIPTION] + return data["description"] def column_participant(self, data): - handle = data[0] + handle = data["handle"] cached, value = self.get_cached_value(handle, "PARTICIPANT") if not cached: value = get_participant_from_event( - self.db, data[COLUMN_HANDLE], all_=True + self.db, data["handle"], all_=True ) # all participants self.set_cached_value(handle, "PARTICIPANT", value) return value def column_place(self, data): - if data[COLUMN_PLACE]: - cached, value = self.get_cached_value(data[0], "PLACE") + if data["place"]: + cached, value = self.get_cached_value(data["handle"], "PLACE") if not cached: - event = Event() - event.unserialize(data) + event = from_dict(data) value = place_displayer.display_event(self.db, event) - self.set_cached_value(data[0], "PLACE", value) + self.set_cached_value(data["handle"], "PLACE", value) return value else: return "" def column_type(self, data): - return str(EventType(data[COLUMN_TYPE])) + return str(EventType(data["type"])) def column_id(self, data): - return data[COLUMN_ID] + return data["gramps_id"] def column_date(self, data): - if data[COLUMN_DATE]: - event = Event() - event.unserialize(data) + if data["date"]: + event = from_dict(data) date_str = get_date(event) if date_str != "": retval = escape(date_str) @@ -180,9 +165,8 @@ def column_date(self, data): return "" def sort_date(self, data): - if data[COLUMN_DATE]: - event = Event() - event.unserialize(data) + if data["date"]: + event = from_dict(data) retval = "%09d" % event.get_date_object().get_sort_value() if not get_date_valid(event): return INVALID_DATE_FORMAT % retval @@ -192,17 +176,17 @@ def sort_date(self, data): return "" def column_private(self, data): - if data[COLUMN_PRIV]: + if data["private"]: return "gramps-lock" else: # There is a problem returning None here. return "" def sort_change(self, data): - return "%012x" % data[COLUMN_CHANGE] + return "%012x" % data["change"] def column_change(self, data): - return format_time(data[COLUMN_CHANGE]) + return format_time(data["change"]) def get_tag_name(self, tag_handle): """ @@ -219,12 +203,12 @@ def column_tag_color(self, data): """ Return the tag color. """ - tag_handle = data[0] + tag_handle = data["handle"] cached, tag_color = self.get_cached_value(tag_handle, "TAG_COLOR") if not cached: tag_color = "" tag_priority = None - for handle in data[COLUMN_TAGS]: + for handle in data["tag_list"]: tag = self.db.get_tag_from_handle(handle) this_priority = tag.get_priority() if tag_priority is None or this_priority < tag_priority: @@ -237,6 +221,6 @@ def column_tags(self, data): """ Return the sorted list of tags. """ - tag_list = list(map(self.get_tag_name, data[COLUMN_TAGS])) + tag_list = list(map(self.get_tag_name, data["tag_list"])) # TODO for Arabic, should the next line's comma be translated? return ", ".join(sorted(tag_list, key=glocale.sort_key)) diff --git a/gramps/gui/views/treemodels/familymodel.py b/gramps/gui/views/treemodels/familymodel.py index aff4c6ef828..6204036cd13 100644 --- a/gramps/gui/views/treemodels/familymodel.py +++ b/gramps/gui/views/treemodels/familymodel.py @@ -3,6 +3,7 @@ # # Copyright (C) 2000-2007 Donald N. Allingham # Copyright (C) 2010 Nick Hall +# Copyright (C) 2024 Doug Blank # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -116,11 +117,11 @@ def on_get_n_columns(self): return len(self.fmap) + 1 def column_father(self, data): - handle = data[0] + handle = data["handle"] cached, value = self.get_cached_value(handle, "FATHER") if not cached: - if data[2]: - person = self.db.get_person_from_handle(data[2]) + if data["father_handle"]: + person = self.db.get_person_from_handle(data["father_handle"]) value = name_displayer.display_name(person.primary_name) else: value = "" @@ -128,11 +129,11 @@ def column_father(self, data): return value def sort_father(self, data): - handle = data[0] + handle = data["handle"] cached, value = self.get_cached_value(handle, "SORT_FATHER") if not cached: - if data[2]: - person = self.db.get_person_from_handle(data[2]) + if data["father_handle"]: + person = self.db.get_person_from_handle(data["father_handle"]) value = name_displayer.sorted_name(person.primary_name) else: value = "" @@ -140,11 +141,11 @@ def sort_father(self, data): return value def column_mother(self, data): - handle = data[0] + handle = data["handle"] cached, value = self.get_cached_value(handle, "MOTHER") if not cached: - if data[3]: - person = self.db.get_person_from_handle(data[3]) + if data["mother_handle"]: + person = self.db.get_person_from_handle(data["mother_handle"]) value = name_displayer.display_name(person.primary_name) else: value = "" @@ -152,11 +153,11 @@ def column_mother(self, data): return value def sort_mother(self, data): - handle = data[0] + handle = data["handle"] cached, value = self.get_cached_value(handle, "SORT_MOTHER") if not cached: - if data[3]: - person = self.db.get_person_from_handle(data[3]) + if data["mother_handle"]: + person = self.db.get_person_from_handle(data["mother_handle"]) value = name_displayer.sorted_name(person.primary_name) else: value = "" @@ -164,15 +165,15 @@ def sort_mother(self, data): return value def column_type(self, data): - return str(FamilyRelType(data[5])) + return data["type"]["string"] def column_marriage(self, data): - handle = data[0] + handle = data["handle"] cached, value = self.get_cached_value(handle, "MARRIAGE") if not cached: - family = self.db.get_family_from_handle(data[0]) + family = self.db.get_family_from_handle(data["handle"]) event = get_marriage_or_fallback(self.db, family, "%s") - if event: + if event and event.date: if event.date.format: value = event.date.format % displayer.display(event.date) elif not get_date_valid(event): @@ -185,10 +186,10 @@ def column_marriage(self, data): return value def sort_marriage(self, data): - handle = data[0] + handle = data["handle"] cached, value = self.get_cached_value(handle, "SORT_MARRIAGE") if not cached: - family = self.db.get_family_from_handle(data[0]) + family = self.db.get_family_from_handle(data["handle"]) event = get_marriage_or_fallback(self.db, family) if event: value = "%09d" % event.date.get_sort_value() @@ -198,20 +199,20 @@ def sort_marriage(self, data): return value def column_id(self, data): - return data[1] + return data["gramps_id"] def column_private(self, data): - if data[14]: + if data["private"]: return "gramps-lock" else: # There is a problem returning None here. return "" def sort_change(self, data): - return "%012x" % data[12] + return "%012x" % data["change"] def column_change(self, data): - return format_time(data[12]) + return format_time(data["change"]) def get_tag_name(self, tag_handle): """ @@ -227,12 +228,12 @@ def column_tag_color(self, data): """ Return the tag color. """ - tag_handle = data[0] + tag_handle = data["handle"] cached, tag_color = self.get_cached_value(tag_handle, "TAG_COLOR") if not cached: tag_color = "" tag_priority = None - for handle in data[13]: + for handle in data["tag_list"]: tag = self.db.get_tag_from_handle(handle) this_priority = tag.get_priority() if tag_priority is None or this_priority < tag_priority: @@ -245,6 +246,6 @@ def column_tags(self, data): """ Return the sorted list of tags. """ - tag_list = list(map(self.get_tag_name, data[13])) + tag_list = list(map(self.get_tag_name, data["tag_list"])) # TODO for Arabic, should the next line's comma be translated? return ", ".join(sorted(tag_list, key=glocale.sort_key)) diff --git a/gramps/gui/views/treemodels/mediamodel.py b/gramps/gui/views/treemodels/mediamodel.py index 0663010aff0..188bb9262f4 100644 --- a/gramps/gui/views/treemodels/mediamodel.py +++ b/gramps/gui/views/treemodels/mediamodel.py @@ -3,6 +3,7 @@ # # Copyright (C) 2000-2006 Donald N. Allingham # Copyright (C) 2010 Nick Hall +# Copyright (C) 2024 Doug Blank # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -45,6 +46,7 @@ _ = glocale.translation.gettext from gramps.gen.datehandler import displayer, format_time from gramps.gen.lib import Date, Media +from gramps.gen.lib.serialize import from_dict from .flatbasemodel import FlatBaseModel @@ -109,37 +111,36 @@ def color_column(self): """ Return the color column. """ + # For model.get_value() arg return 8 def on_get_n_columns(self): return len(self.fmap) + 1 def column_description(self, data): - return data[4] + return data["desc"] def column_path(self, data): - return data[2] + return data["path"] def column_mime(self, data): - mime = data[3] + mime = data["mime"] if mime: return mime else: return _("Note") def column_id(self, data): - return data[1] + return data["gramps_id"] def column_date(self, data): - if data[10]: - date = Date() - date.unserialize(data[10]) + if data["date"]: + date = from_dict(data["date"]) return displayer.display(date) return "" def sort_date(self, data): - obj = Media() - obj.unserialize(data) + obj = from_dict(data) d = obj.get_date_object() if d: return "%09d" % d.get_sort_value() @@ -147,20 +148,20 @@ def sort_date(self, data): return "" def column_handle(self, data): - return str(data[0]) + return str(data["handle"]) def column_private(self, data): - if data[12]: + if data["private"]: return "gramps-lock" else: # There is a problem returning None here. return "" def sort_change(self, data): - return "%012x" % data[9] + return "%012x" % data["change"] def column_change(self, data): - return format_time(data[9]) + return format_time(data["change"]) def column_tooltip(self, data): return "Media tooltip" @@ -179,12 +180,12 @@ def column_tag_color(self, data): """ Return the tag color. """ - tag_handle = data[0] + tag_handle = data["handle"] cached, tag_color = self.get_cached_value(tag_handle, "TAG_COLOR") if not cached: tag_color = "" tag_priority = None - for handle in data[11]: + for handle in data["tag_list"]: tag = self.db.get_tag_from_handle(handle) this_priority = tag.get_priority() if tag_priority is None or this_priority < tag_priority: @@ -197,6 +198,6 @@ def column_tags(self, data): """ Return the sorted list of tags. """ - tag_list = list(map(self.get_tag_name, data[11])) + tag_list = list(map(self.get_tag_name, data["tag_list"])) # TODO for Arabic, should the next line's comma be translated? return ", ".join(sorted(tag_list, key=glocale.sort_key)) diff --git a/gramps/gui/views/treemodels/notemodel.py b/gramps/gui/views/treemodels/notemodel.py index 3aa149b035a..c3b920d2f36 100644 --- a/gramps/gui/views/treemodels/notemodel.py +++ b/gramps/gui/views/treemodels/notemodel.py @@ -3,6 +3,7 @@ # # Copyright (C) 2000-2007 Donald N. Allingham # Copyright (C) 2010 Nick Hall +# Copyright (C) 2024 Doug Blank # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -104,6 +105,7 @@ def color_column(self): """ Return the color column. """ + # This is for model.get_value() arg return 6 def on_get_n_columns(self): @@ -112,17 +114,17 @@ def on_get_n_columns(self): def column_id(self, data): """Return the id of the Note.""" - return data[Note.POS_ID] + return data["gramps_id"] def column_type(self, data): """Return the type of the Note in readable format.""" temp = NoteType() - temp.set(data[Note.POS_TYPE]) + temp.set(data["type"]) return str(temp) def column_preview(self, data): """Return a shortend version of the Note's text.""" - note = data[Note.POS_TEXT][StyledText.POS_TEXT] + note = data["text"]["string"] note = " ".join(note.split()) if len(note) > 80: return note[:80] + "..." @@ -130,17 +132,17 @@ def column_preview(self, data): return note def column_private(self, data): - if data[Note.POS_PRIVATE]: + if data["private"]: return "gramps-lock" else: # There is a problem returning None here. return "" def sort_change(self, data): - return "%012x" % data[Note.POS_CHANGE] + return "%012x" % data["change"] def column_change(self, data): - return format_time(data[Note.POS_CHANGE]) + return format_time(data["change"]) def get_tag_name(self, tag_handle): """ @@ -156,12 +158,12 @@ def column_tag_color(self, data): """ Return the tag color. """ - tag_handle = data[0] + tag_handle = data["handle"] cached, value = self.get_cached_value(tag_handle, "TAG_COLOR") if not cached: tag_color = "" tag_priority = None - for handle in data[Note.POS_TAGS]: + for handle in data["tag_list"]: tag = self.db.get_tag_from_handle(handle) if tag: this_priority = tag.get_priority() @@ -176,6 +178,6 @@ def column_tags(self, data): """ Return the sorted list of tags. """ - tag_list = list(map(self.get_tag_name, data[Note.POS_TAGS])) + tag_list = list(map(self.get_tag_name, data["tag_list"])) # TODO for Arabic, should the next line's comma be translated? return ", ".join(sorted(tag_list, key=glocale.sort_key)) diff --git a/gramps/gui/views/treemodels/peoplemodel.py b/gramps/gui/views/treemodels/peoplemodel.py index fefd6e39c46..cb904c561ca 100644 --- a/gramps/gui/views/treemodels/peoplemodel.py +++ b/gramps/gui/views/treemodels/peoplemodel.py @@ -5,6 +5,7 @@ # Copyright (C) 2009 Gary Burton # Copyright (C) 2009-2010 Nick Hall # Copyright (C) 2009 Benny Malengier +# Copyright (C) 2024 Doug Blank # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -66,6 +67,7 @@ ChildRefType, NoteType, ) +from gramps.gen.lib.serialize import from_dict from gramps.gen.display.name import displayer as name_displayer from gramps.gen.display.place import displayer as place_displayer from gramps.gen.datehandler import format_time, get_date, get_date_valid @@ -74,23 +76,6 @@ from .basemodel import BaseModel from gramps.gen.config import config -# ------------------------------------------------------------------------- -# -# COLUMN constants; positions in raw data structure -# -# ------------------------------------------------------------------------- -COLUMN_ID = 1 -COLUMN_GENDER = 2 -COLUMN_NAME = 3 -COLUMN_DEATH = 5 -COLUMN_BIRTH = 6 -COLUMN_EVENT = 7 -COLUMN_FAMILY = 8 -COLUMN_PARENT = 9 -COLUMN_NOTES = 16 -COLUMN_CHANGE = 17 -COLUMN_TAGS = 18 -COLUMN_PRIV = 19 invalid_date_format = config.get("preferences.invalid-date-format") no_surname = config.get("preferences.no-surname-text") @@ -176,23 +161,23 @@ def on_get_n_columns(self): return len(self.fmap) + 1 def sort_name(self, data): - handle = data[0] + handle = data["handle"] cached, name = self.get_cached_value(handle, "SORT_NAME") if not cached: - name = name_displayer.raw_sorted_name(data[COLUMN_NAME]) + name = name_displayer.raw_sorted_name(data["primary_name"]) self.set_cached_value(handle, "SORT_NAME", name) return name def column_name(self, data): - handle = data[0] + handle = data["handle"] cached, name = self.get_cached_value(handle, "NAME") if not cached: - name = name_displayer.raw_display_name(data[COLUMN_NAME]) + name = name_displayer.raw_display_name(data["primary_name"]) self.set_cached_value(handle, "NAME", name) return name def column_spouse(self, data): - handle = data[0] + handle = data["handle"] cached, value = self.get_cached_value(handle, "SPOUSE") if not cached: value = self._get_spouse_data(data) @@ -200,7 +185,7 @@ def column_spouse(self, data): return value def column_private(self, data): - if data[COLUMN_PRIV]: + if data["private"]: return "gramps-lock" else: # There is a problem returning None here. @@ -208,12 +193,12 @@ def column_private(self, data): def _get_spouse_data(self, data): spouses_names = "" - for family_handle in data[COLUMN_FAMILY]: + for family_handle in data["family_list"]: family = self.db.get_family_from_handle(family_handle) for spouse_id in [family.get_father_handle(), family.get_mother_handle()]: if not spouse_id: continue - if spouse_id == data[0]: + if spouse_id == data["handle"]: continue spouse = self.db.get_person_from_handle(spouse_id) if spouses_names: @@ -222,19 +207,19 @@ def _get_spouse_data(self, data): return spouses_names def column_id(self, data): - return data[COLUMN_ID] + return data["gramps_id"] def sort_change(self, data): - return "%012x" % data[COLUMN_CHANGE] + return "%012x" % data["change"] def column_change(self, data): - return format_time(data[COLUMN_CHANGE]) + return format_time(data["change"]) def column_gender(self, data): - return PeopleBaseModel._GENDER[data[COLUMN_GENDER]] + return PeopleBaseModel._GENDER[data["gender"]] def column_birth_day(self, data): - handle = data[0] + handle = data["handle"] cached, value = self.get_cached_value(handle, "BIRTH_DAY") if not cached: value = self._get_birth_data(data, False) @@ -242,7 +227,7 @@ def column_birth_day(self, data): return value def sort_birth_day(self, data): - handle = data[0] + handle = data["handle"] cached, value = self.get_cached_value(handle, "SORT_BIRTH_DAY") if not cached: value = self._get_birth_data(data, True) @@ -250,12 +235,11 @@ def sort_birth_day(self, data): return value def _get_birth_data(self, data, sort_mode): - index = data[COLUMN_BIRTH] + index = data["birth_ref_index"] if index != -1: try: - local = data[COLUMN_EVENT][index] - b = EventRef() - b.unserialize(local) + local = data["event_ref_list"][index] + b = from_dict(local) birth = self.db.get_event_from_handle(b.ref) if sort_mode: retval = "%09d" % birth.get_date_object().get_sort_value() @@ -270,9 +254,8 @@ def _get_birth_data(self, data, sort_mode): except: return "" - for event_ref in data[COLUMN_EVENT]: - er = EventRef() - er.unserialize(event_ref) + for event_ref in data["event_ref_list"]: + er = from_dict(event_ref) event = self.db.get_event_from_handle(er.ref) etype = event.get_type() date_str = get_date(event) @@ -293,7 +276,7 @@ def _get_birth_data(self, data, sort_mode): return "" def column_death_day(self, data): - handle = data[0] + handle = data["handle"] cached, value = self.get_cached_value(handle, "DEATH_DAY") if not cached: value = self._get_death_data(data, False) @@ -301,7 +284,7 @@ def column_death_day(self, data): return value def sort_death_day(self, data): - handle = data[0] + handle = data["handle"] cached, value = self.get_cached_value(handle, "SORT_DEATH_DAY") if not cached: value = self._get_death_data(data, True) @@ -309,12 +292,11 @@ def sort_death_day(self, data): return value def _get_death_data(self, data, sort_mode): - index = data[COLUMN_DEATH] + index = data["death_ref_index"] if index != -1: try: - local = data[COLUMN_EVENT][index] - ref = EventRef() - ref.unserialize(local) + local = data["event_ref_list"][index] + ref = from_dict(local) event = self.db.get_event_from_handle(ref.ref) if sort_mode: retval = "%09d" % event.get_date_object().get_sort_value() @@ -329,9 +311,8 @@ def _get_death_data(self, data, sort_mode): except: return "" - for event_ref in data[COLUMN_EVENT]: - er = EventRef() - er.unserialize(event_ref) + for event_ref in data["event_ref_list"]: + er = from_dict(event_ref) event = self.db.get_event_from_handle(er.ref) etype = event.get_type() date_str = get_date(event) @@ -351,17 +332,16 @@ def _get_death_data(self, data, sort_mode): return "" def column_birth_place(self, data): - handle = data[0] + handle = data["handle"] cached, value = self.get_cached_value(handle, "BIRTH_PLACE") if cached: return value else: - index = data[COLUMN_BIRTH] + index = data["birth_ref_index"] if index != -1: try: - local = data[COLUMN_EVENT][index] - br = EventRef() - br.unserialize(local) + local = data["event_ref_list"][index] + br = from_dict(local) event = self.db.get_event_from_handle(br.ref) if event: place_title = place_displayer.display_event(self.db, event) @@ -374,9 +354,8 @@ def column_birth_place(self, data): self.set_cached_value(handle, "BIRTH_PLACE", value) return value - for event_ref in data[COLUMN_EVENT]: - er = EventRef() - er.unserialize(event_ref) + for event_ref in data["event_ref_list"]: + er = from_dict(event_ref) event = self.db.get_event_from_handle(er.ref) etype = event.get_type() if etype.is_birth_fallback() and er.get_role() == EventRoleType.PRIMARY: @@ -390,17 +369,16 @@ def column_birth_place(self, data): return value def column_death_place(self, data): - handle = data[0] + handle = data["handle"] cached, value = self.get_cached_value(handle, "DEATH_PLACE") if cached: return value else: - index = data[COLUMN_DEATH] + index = data["death_ref_index"] if index != -1: try: - local = data[COLUMN_EVENT][index] - dr = EventRef() - dr.unserialize(local) + local = data["event_ref_list"][index] + dr = from_dict(local) event = self.db.get_event_from_handle(dr.ref) if event: place_title = place_displayer.display_event(self.db, event) @@ -413,9 +391,8 @@ def column_death_place(self, data): self.set_cached_value(handle, "DEATH_PLACE", value) return value - for event_ref in data[COLUMN_EVENT]: - er = EventRef() - er.unserialize(event_ref) + for event_ref in data["event_ref_list"]: + er = from_dict(event_ref) event = self.db.get_event_from_handle(er.ref) etype = event.get_type() if etype.is_death_fallback() and er.get_role() == EventRoleType.PRIMARY: @@ -430,8 +407,8 @@ def column_death_place(self, data): def _get_parents_data(self, data): parents = 0 - if data[COLUMN_PARENT]: - person = self.db.get_person_from_gramps_id(data[COLUMN_ID]) + if data["parent_family_list"]: + person = self.db.get_person_from_gramps_id(data["gramps_id"]) family_list = person.get_parent_family_handle_list() for fam_hdle in family_list: family = self.db.get_family_from_handle(fam_hdle) @@ -443,7 +420,7 @@ def _get_parents_data(self, data): def _get_marriages_data(self, data): marriages = 0 - for family_handle in data[COLUMN_FAMILY]: + for family_handle in data["family_list"]: family = self.db.get_family_from_handle(family_handle) if int(family.get_relationship()) == FamilyRelType.MARRIED: marriages += 1 @@ -451,7 +428,7 @@ def _get_marriages_data(self, data): def _get_children_data(self, data): children = 0 - for family_handle in data[COLUMN_FAMILY]: + for family_handle in data["family_list"]: family = self.db.get_family_from_handle(family_handle) for child_ref in family.get_child_ref_list(): if ( @@ -463,14 +440,14 @@ def _get_children_data(self, data): def _get_todo_data(self, data): todo = 0 - for note_handle in data[COLUMN_NOTES]: + for note_handle in data["note_list"]: note = self.db.get_note_from_handle(note_handle) if int(note.get_type()) == NoteType.TODO: todo += 1 return todo def column_parents(self, data): - handle = data[0] + handle = data["handle"] cached, value = self.get_cached_value(handle, "PARENTS") if not cached: value = self._get_parents_data(data) @@ -478,7 +455,7 @@ def column_parents(self, data): return str(value) def sort_parents(self, data): - handle = data[0] + handle = data["handle"] cached, value = self.get_cached_value(handle, "SORT_PARENTS") if not cached: value = self._get_parents_data(data) @@ -486,7 +463,7 @@ def sort_parents(self, data): return "%06d" % value def column_marriages(self, data): - handle = data[0] + handle = data["handle"] cached, value = self.get_cached_value(handle, "MARRIAGES") if not cached: value = self._get_marriages_data(data) @@ -494,7 +471,7 @@ def column_marriages(self, data): return str(value) def sort_marriages(self, data): - handle = data[0] + handle = data["handle"] cached, value = self.get_cached_value(handle, "SORT_MARRIAGES") if not cached: value = self._get_marriages_data(data) @@ -502,7 +479,7 @@ def sort_marriages(self, data): return "%06d" % value def column_children(self, data): - handle = data[0] + handle = data["handle"] cached, value = self.get_cached_value(handle, "CHILDREN") if not cached: value = self._get_children_data(data) @@ -510,7 +487,7 @@ def column_children(self, data): return str(value) def sort_children(self, data): - handle = data[0] + handle = data["handle"] cached, value = self.get_cached_value(handle, "SORT_CHILDREN") if not cached: value = self._get_children_data(data) @@ -518,7 +495,7 @@ def sort_children(self, data): return "%06d" % value def column_todo(self, data): - handle = data[0] + handle = data["handle"] cached, value = self.get_cached_value(handle, "TODO") if not cached: value = self._get_todo_data(data) @@ -526,7 +503,7 @@ def column_todo(self, data): return str(value) def sort_todo(self, data): - handle = data[0] + handle = data["handle"] cached, value = self.get_cached_value(handle, "SORT_TODO") if not cached: value = self._get_todo_data(data) @@ -549,12 +526,12 @@ def column_tag_color(self, data): """ Return the tag color. """ - tag_handle = data[0] + tag_handle = data["handle"] cached, value = self.get_cached_value(tag_handle, "TAG_COLOR") if not cached: tag_color = "" tag_priority = None - for handle in data[COLUMN_TAGS]: + for handle in data["tag_list"]: tag = self.db.get_tag_from_handle(handle) if tag: this_priority = tag.get_priority() @@ -569,10 +546,10 @@ def column_tags(self, data): """ Return the sorted list of tags. """ - handle = data[0] + handle = data["handle"] cached, value = self.get_cached_value(handle, "TAGS") if not cached: - tag_list = list(map(self.get_tag_name, data[COLUMN_TAGS])) + tag_list = list(map(self.get_tag_name, data["tag_list"])) # TODO for Arabic, should the next line's comma be translated? value = ", ".join(sorted(tag_list, key=glocale.sort_key)) self.set_cached_value(handle, "TAGS", value) @@ -671,7 +648,7 @@ def add_row(self, handle, data): """ ngn = name_displayer.name_grouping_data - name_data = data[COLUMN_NAME] + name_data = data["primary_name"] group_name = ngn(self.db, name_data) sort_key = self.sort_func(data) diff --git a/gramps/gui/views/treemodels/placemodel.py b/gramps/gui/views/treemodels/placemodel.py index 4b1adffd698..07c1129f6ab 100644 --- a/gramps/gui/views/treemodels/placemodel.py +++ b/gramps/gui/views/treemodels/placemodel.py @@ -5,6 +5,7 @@ # Copyright (C) 2009-2010 Nick Hall # Copyright (C) 2009 Benny Malengier # Copyright (C) 2010 Gary Burton +# Copyright (C) 2024 Doug Blank # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -46,6 +47,7 @@ # # ------------------------------------------------------------------------- from gramps.gen.lib import Place, PlaceType +from gramps.gen.lib.serialize import from_dict from gramps.gen.datehandler import format_time from gramps.gen.utils.place import conv_lat_lon, coord_formats from gramps.gen.display.place import displayer as place_displayer @@ -115,86 +117,95 @@ def color_column(self): """ Return the color column. """ + # This is a model.get_value() arg return 10 def on_get_n_columns(self): return len(self.fmap) + 1 def column_title(self, data): - handle = data[0] + handle = data["handle"] cached, value = self.get_cached_value(handle, "PLACE") if not cached: - place = Place() - place.unserialize(data) + place = from_dict(data) value = place_displayer.display(self.db, place) self.set_cached_value(handle, "PLACE", value) return value def column_name(self, data): """Return the primary name""" - return data[6][0] + return data["name"]["value"] def search_name(self, data): """The search name includes all alt names to enable finding by alt name""" - return ",".join([data[6][0]] + [name[0] for name in data[7]]) + return ",".join( + [data["name"]["value"]] + [name["value"] for name in data["alt_name"]] + ) def column_longitude(self, data): - if not data[3]: + if not data["long"]: return "" value = conv_lat_lon( - "0", data[3], format=coord_formats[config.get("preferences.coord-format")] + "0", + data["long"], + format=coord_formats[config.get("preferences.coord-format")], )[1] if not value: return _("Error in format") return ("\u202d" + value + "\u202e") if glocale.rtl_locale else value def column_latitude(self, data): - if not data[4]: + if not data["lat"]: return "" value = conv_lat_lon( - data[4], "0", format=coord_formats[config.get("preferences.coord-format")] + data["lat"], + "0", + format=coord_formats[config.get("preferences.coord-format")], )[0] if not value: return _("Error in format") return ("\u202d" + value + "\u202e") if glocale.rtl_locale else value def sort_longitude(self, data): - if not data[3]: + if not data["long"]: return "" - value = conv_lat_lon("0", data[3], format="ISO-DMS") if data[3] else "" + value = ( + conv_lat_lon("0", data["long"], format="ISO-DMS") if data["long"] else "" + ) if not value: return _("Error in format") return value def sort_latitude(self, data): - if not data[4]: + if not data["lat"]: return "" - value = conv_lat_lon(data[4], "0", format="ISO-DMS") if data[4] else "" + value = conv_lat_lon(data["lat"], "0", format="ISO-DMS") if data["lat"] else "" if not value: return _("Error in format") return value def column_id(self, data): - return data[1] + return data["gramps_id"] def column_type(self, data): - return str(PlaceType(data[8])) + pt = from_dict(data["place_type"]) + return str(pt) def column_code(self, data): - return data[9] + return data["code"] def column_private(self, data): - if data[17]: + if data["private"]: return "gramps-lock" else: # There is a problem returning None here. return "" def sort_change(self, data): - return "%012x" % data[15] + return "%012x" % data["change"] def column_change(self, data): - return format_time(data[15]) + return format_time(data["change"]) def get_tag_name(self, tag_handle): """ @@ -210,12 +221,12 @@ def column_tag_color(self, data): """ Return the tag color. """ - tag_handle = data[0] + tag_handle = data["handle"] cached, value = self.get_cached_value(tag_handle, "TAG_COLOR") if not cached: tag_color = "" tag_priority = None - for handle in data[16]: + for handle in data["tag_list"]: tag = self.db.get_tag_from_handle(handle) if tag: this_priority = tag.get_priority() @@ -230,7 +241,7 @@ def column_tags(self, data): """ Return the sorted list of tags. """ - tag_list = list(map(self.get_tag_name, data[16])) + tag_list = list(map(self.get_tag_name, data["tag_list"])) # TODO for Arabic, should the next line's comma be translated? return ", ".join(sorted(tag_list, key=glocale.sort_key)) @@ -351,8 +362,8 @@ def add_row(self, handle, data): data The object data. """ sort_key = self.sort_func(data) - if len(data[5]) > 0: - parent = data[5][0][0] + if len(data["placeref_list"]) > 0: + parent = data["placeref_list"][0]["ref"] else: parent = None diff --git a/gramps/gui/views/treemodels/repomodel.py b/gramps/gui/views/treemodels/repomodel.py index 1fb8df8a718..e42112c5a4d 100644 --- a/gramps/gui/views/treemodels/repomodel.py +++ b/gramps/gui/views/treemodels/repomodel.py @@ -2,6 +2,7 @@ # Gramps - a GTK+/GNOME based genealogy program # # Copyright (C) 2000-2006 Donald N. Allingham +# Copyright (C) 2024 Doug Blank # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -40,6 +41,7 @@ # # ------------------------------------------------------------------------- from gramps.gen.lib import Address, RepositoryType, Url, UrlType +from gramps.gen.lib.serialize import from_dict from gramps.gen.datehandler import format_time from .flatbasemodel import FlatBaseModel from gramps.gen.const import GRAMPS_LOCALE as glocale @@ -122,25 +124,25 @@ def color_column(self): """ Return the color column. """ + # For model.get_value() arg return 15 def on_get_n_columns(self): return len(self.fmap) + 1 def column_id(self, data): - return data[1] + return data["gramps_id"] def column_type(self, data): - return str(RepositoryType(data[2])) + return str(RepositoryType(data["type"])) def column_name(self, data): - return data[3] + return data["name"] def column_city(self, data): try: - if data[5]: - addr = Address() - addr.unserialize(data[5][0]) + if data["address_list"]: + addr = from_dict(data["address_list"][0]) return addr.get_city() except: pass @@ -148,9 +150,8 @@ def column_city(self, data): def column_street(self, data): try: - if data[5]: - addr = Address() - addr.unserialize(data[5][0]) + if data["address_list"]: + addr = from_dict(data["address_list"][0]) return addr.get_street() except: pass @@ -158,9 +159,8 @@ def column_street(self, data): def column_locality(self, data): try: - if data[5]: - addr = Address() - addr.unserialize(data[5][0]) + if data["address_list"]: + addr = from_dict(data["address_list"][0]) return addr.get_locality() except: pass @@ -168,9 +168,8 @@ def column_locality(self, data): def column_state(self, data): try: - if data[5]: - addr = Address() - addr.unserialize(data[5][0]) + if data["address_list"]: + addr = from_dict(data["address_list"][0]) return addr.get_state() except: pass @@ -178,9 +177,8 @@ def column_state(self, data): def column_country(self, data): try: - if data[5]: - addr = Address() - addr.unserialize(data[5][0]) + if data["address_list"]: + addr = from_dict(data["address_list"][0]) return addr.get_country() except: pass @@ -188,9 +186,8 @@ def column_country(self, data): def column_postal_code(self, data): try: - if data[5]: - addr = Address() - addr.unserialize(data[5][0]) + if data["address_list"]: + addr = from_dict(data["address_list"][0]) return addr.get_postal_code() except: pass @@ -198,53 +195,49 @@ def column_postal_code(self, data): def column_phone(self, data): try: - if data[5]: - addr = Address() - addr.unserialize(data[5][0]) + if data["address_list"]: + addr = from_dict(data["address_list"][0]) return addr.get_phone() except: pass return "" def column_email(self, data): - if data[6]: - for i in data[6]: - url = Url() - url.unserialize(i) + if data["urls"]: + for url_data in data["urls"]: + url = from_dict(url_data) if url.get_type() == UrlType.EMAIL: return url.path return "" def column_search_url(self, data): - if data[6]: - for i in data[6]: - url = Url() - url.unserialize(i) + if data["urls"]: + for url_data in data["urls"]: + url = from_dict(url_data) if url.get_type() == UrlType.WEB_SEARCH: return url.path return "" def column_home_url(self, data): - if data[6]: - for i in data[6]: - url = Url() - url.unserialize(i) + if data["urls"]: + for url_data in data["urls"]: + url = from_dict(url_data) if url.get_type() == UrlType.WEB_HOME: return url.path return "" def column_private(self, data): - if data[9]: + if data["private"]: return "gramps-lock" else: # There is a problem returning None here. return "" def sort_change(self, data): - return "%012x" % data[7] + return "%012x" % data["change"] def column_change(self, data): - return format_time(data[7]) + return format_time(data["change"]) def get_tag_name(self, tag_handle): """ @@ -261,12 +254,12 @@ def column_tag_color(self, data): """ Return the tag color. """ - tag_handle = data[0] + tag_handle = data["handle"] cached, tag_color = self.get_cached_value(tag_handle, "TAG_COLOR") if not cached: tag_color = "" tag_priority = None - for handle in data[8]: + for handle in data["tag_list"]: tag = self.db.get_tag_from_handle(handle) this_priority = tag.get_priority() if tag_priority is None or this_priority < tag_priority: @@ -279,6 +272,6 @@ def column_tags(self, data): """ Return the sorted list of tags. """ - tag_list = list(map(self.get_tag_name, data[8])) + tag_list = list(map(self.get_tag_name, data["tag_list"])) # TODO for Arabic, should the next line's comma be translated? return ", ".join(sorted(tag_list, key=glocale.sort_key)) diff --git a/gramps/gui/views/treemodels/sourcemodel.py b/gramps/gui/views/treemodels/sourcemodel.py index fb276a321eb..2012a5339de 100644 --- a/gramps/gui/views/treemodels/sourcemodel.py +++ b/gramps/gui/views/treemodels/sourcemodel.py @@ -2,6 +2,7 @@ # Gramps - a GTK+/GNOME based genealogy program # # Copyright (C) 2000-2006 Donald N. Allingham +# Copyright (C) 2024 Doug Blank # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -103,38 +104,39 @@ def color_column(self): """ Return the color column. """ + # This is for model.get_value() arg return 8 def on_get_n_columns(self): return len(self.fmap) + 1 def column_title(self, data): - return data[2].replace("\n", " ") + return data["title"].replace("\n", " ") def column_author(self, data): - return data[3] + return data["author"] def column_abbrev(self, data): - return data[7] + return data["abbrev"] def column_id(self, data): - return data[1] + return data["gramps_id"] def column_pubinfo(self, data): - return data[4] + return data["pubinfo"] def column_private(self, data): - if data[12]: + if data["private"]: return "gramps-lock" else: # There is a problem returning None here. return "" def column_change(self, data): - return format_time(data[8]) + return format_time(data["change"]) def sort_change(self, data): - return "%012x" % data[8] + return "%012x" % data["change"] def get_tag_name(self, tag_handle): """ @@ -150,12 +152,12 @@ def column_tag_color(self, data): """ Return the tag color. """ - tag_handle = data[0] + tag_handle = data["handle"] cached, value = self.get_cached_value(tag_handle, "TAG_COLOR") if not cached: tag_color = "" tag_priority = None - for handle in data[11]: + for handle in data["tag_list"]: tag = self.db.get_tag_from_handle(handle) if tag: this_priority = tag.get_priority() @@ -170,6 +172,6 @@ def column_tags(self, data): """ Return the sorted list of tags. """ - tag_list = list(map(self.get_tag_name, data[11])) + tag_list = list(map(self.get_tag_name, data["tag_list"])) # TODO for Arabic, should the next line's comma be translated? return ", ".join(sorted(tag_list, key=glocale.sort_key)) diff --git a/gramps/plugins/db/bsddb/bsddb.py b/gramps/plugins/db/bsddb/bsddb.py index 6cb45b577ee..0f91009aa08 100644 --- a/gramps/plugins/db/bsddb/bsddb.py +++ b/gramps/plugins/db/bsddb/bsddb.py @@ -2,7 +2,8 @@ # Gramps - a GTK+/GNOME based genealogy program # # Copyright (C) 2020 Paul Culley -# Copyright (C) 2020 Nick Hall +# Copyright (C) 2020 Nick Hall +# Copyright (C) 2024 Doug Blank # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -108,8 +109,11 @@ def load( username=username, password=password, ) + # Default new DBAPI uses "json" + # So, we force read/write in blobs: + self.set_serializer("blob") - # now read in the bsddb and copy to dpapi + # now read in the bsddb and copy to dbapi schema_vers = None total = 0 tables = ( @@ -149,12 +153,6 @@ def load( self.set_total(total) # copy data from each dbmap to sqlite table for old_t, new_t, dbmap in table_list: - self._txn_begin() - if new_t == "metadata": - sql = "REPLACE INTO metadata (setting, value) VALUES " "(?, ?)" - else: - sql = "INSERT INTO %s (handle, blob_data) VALUES " "(?, ?)" % new_t - for key in dbmap.keys(): self.update() data = pickle.loads(dbmap[key], encoding="utf-8") @@ -215,7 +213,15 @@ def load( # These are list, but need to be set data = set(data) - self.dbapi.execute(sql, [key.decode("utf-8"), pickle.dumps(data)]) + self._set_metadata(key.decode("utf-8"), data) + else: + # Not metadata, but object + self._txn_begin() + self.dbapi( + f"INSERT INTO {new_t} (handle, blob_data) VALUES " "(?, ?)", + [key.decode("utf-8"), pickle.dumps(data)], + ) + self._txn_commit() # get schema version from file if not in metadata if new_t == "metadata" and schema_vers is None: @@ -226,8 +232,8 @@ def load( else: schema_vers = 0 # and put schema version into metadata - self.dbapi.execute(sql, ["version", schema_vers]) - self._txn_commit() + self._set_metadata("version", schema_vers) + dbmap.close() if new_t == "metadata" and schema_vers < _MINVERSION: raise DbVersionError(schema_vers, _MINVERSION, _DBVERSION) diff --git a/gramps/plugins/db/dbapi/dbapi.py b/gramps/plugins/db/dbapi/dbapi.py index f39535e5733..60f49362ea2 100644 --- a/gramps/plugins/db/dbapi/dbapi.py +++ b/gramps/plugins/db/dbapi/dbapi.py @@ -1,8 +1,8 @@ # # Gramps - a GTK+/GNOME based genealogy program # -# Copyright (C) 2015-2016 Douglas S. Blank -# Copyright (C) 2016-2017 Nick Hall +# Copyright (C) 2015-2016,2024 Douglas S. Blank +# Copyright (C) 2016-2017 Nick Hall # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -29,7 +29,7 @@ # # ------------------------------------------------------------------------- import logging -import pickle +import json import time from gramps.gen.const import GRAMPS_LOCALE as glocale @@ -61,6 +61,7 @@ Source, Tag, ) +from gramps.gen.lib.serialize import from_dict, to_dict from gramps.gen.lib.genderstats import GenderStats from gramps.gen.updatecallback import UpdateCallback @@ -81,6 +82,23 @@ class DBAPI(DbGeneric): def _initialize(self, directory, username, password): raise NotImplementedError + def use_json_data(self): + """ + A DBAPI level method for testing if the + database supports JSON access. + """ + # Check if json_data exists on metadata as a proxy to see + # if the database has been converted to use JSON data + return self.dbapi.column_exists("metadata", "json_data") + + def upgrade_table_for_json_data(self, table_name): + """ + A DBAPI level method for upgrading the given table + adding a json_data column. + """ + if not self.dbapi.column_exists(table_name, "json_data"): + self.dbapi.execute("ALTER TABLE %s ADD COLUMN json_data TEXT;" % table_name) + def _schema_exists(self): """ Check to see if the schema exists. @@ -103,42 +121,42 @@ def _create_schema(self): "handle VARCHAR(50) PRIMARY KEY NOT NULL, " "given_name TEXT, " "surname TEXT, " - "blob_data BLOB" + "json_data TEXT" ")" ) self.dbapi.execute( "CREATE TABLE family " "(" "handle VARCHAR(50) PRIMARY KEY NOT NULL, " - "blob_data BLOB" + "json_data TEXT" ")" ) self.dbapi.execute( "CREATE TABLE source " "(" "handle VARCHAR(50) PRIMARY KEY NOT NULL, " - "blob_data BLOB" + "json_data TEXT" ")" ) self.dbapi.execute( "CREATE TABLE citation " "(" "handle VARCHAR(50) PRIMARY KEY NOT NULL, " - "blob_data BLOB" + "json_data TEXT" ")" ) self.dbapi.execute( "CREATE TABLE event " "(" "handle VARCHAR(50) PRIMARY KEY NOT NULL, " - "blob_data BLOB" + "json_data TEXT" ")" ) self.dbapi.execute( "CREATE TABLE media " "(" "handle VARCHAR(50) PRIMARY KEY NOT NULL, " - "blob_data BLOB" + "json_data TEXT" ")" ) self.dbapi.execute( @@ -146,28 +164,28 @@ def _create_schema(self): "(" "handle VARCHAR(50) PRIMARY KEY NOT NULL, " "enclosed_by VARCHAR(50), " - "blob_data BLOB" + "json_data TEXT" ")" ) self.dbapi.execute( "CREATE TABLE repository " "(" "handle VARCHAR(50) PRIMARY KEY NOT NULL, " - "blob_data BLOB" + "json_data TEXT" ")" ) self.dbapi.execute( "CREATE TABLE note " "(" "handle VARCHAR(50) PRIMARY KEY NOT NULL, " - "blob_data BLOB" + "json_data TEXT" ")" ) self.dbapi.execute( "CREATE TABLE tag " "(" "handle VARCHAR(50) PRIMARY KEY NOT NULL, " - "blob_data BLOB" + "json_data TEXT" ")" ) # Secondary: @@ -191,7 +209,7 @@ def _create_schema(self): "CREATE TABLE metadata " "(" "setting VARCHAR(50) PRIMARY KEY NOT NULL, " - "value BLOB" + "json_data TEXT" ")" ) self.dbapi.execute( @@ -344,39 +362,55 @@ def transaction_abort(self, transaction): transaction.last = None self._after_commit(transaction) + def _get_metadata_keys(self): + """ + Get all of the metadata setting names from the + database. + """ + self.dbapi.execute("SELECT setting FROM metadata;") + return [row[0] for row in self.dbapi.fetchall()] + def _get_metadata(self, key, default="_"): """ Get an item from the database. Note we reserve and use _ to denote default value of [] """ - self.dbapi.execute("SELECT value FROM metadata WHERE setting = ?", [key]) + self.dbapi.execute( + f"SELECT {self.serializer.metadata_field} FROM metadata WHERE setting = ?", + [key], + ) row = self.dbapi.fetchone() if row: - return pickle.loads(row[0]) + return self.serializer.metadata_to_object(row[0]) + if default == "_": return [] return default - def _set_metadata(self, key, value): + def _set_metadata(self, key, value, use_txn=True): """ key: string value: item, will be serialized here + + Note: if use_txn, then begin/commit txn """ - self._txn_begin() + if use_txn: + self._txn_begin() self.dbapi.execute("SELECT 1 FROM metadata WHERE setting = ?", [key]) row = self.dbapi.fetchone() if row: self.dbapi.execute( - "UPDATE metadata SET value = ? WHERE setting = ?", - [pickle.dumps(value), key], + f"UPDATE metadata SET {self.serializer.metadata_field} = ? WHERE setting = ?", + [self.serializer.object_to_metadata(value), key], ) else: self.dbapi.execute( - "INSERT INTO metadata (setting, value) VALUES (?, ?)", - [key, pickle.dumps(value)], + f"INSERT INTO metadata (setting, {self.serializer.metadata_field}) VALUES (?, ?)", + [key, self.serializer.object_to_metadata(value)], ) - self._txn_commit() + if use_txn: + self._txn_commit() def get_name_group_keys(self): """ @@ -579,10 +613,12 @@ def get_tag_from_name(self, name): If no such Tag exists, None is returned. """ - self.dbapi.execute("SELECT blob_data FROM tag WHERE name = ?", [name]) + self.dbapi.execute( + f"SELECT {self.serializer.data_field} FROM tag WHERE name = ?", [name] + ) row = self.dbapi.fetchone() if row: - return Tag.create(pickle.loads(row[0])) + return self.serializer.string_to_object(Tag, row[0]) return None def _get_number_of(self, obj_key): @@ -635,22 +671,22 @@ def _commit_base(self, obj, obj_key, trans, change_time): old_data = self._get_raw_data(obj_key, obj.handle) # update the object: self.dbapi.execute( - f"UPDATE {table} SET blob_data = ? WHERE handle = ?", - [pickle.dumps(obj.serialize()), obj.handle], + f"UPDATE {table} SET {self.serializer.data_field} = ? WHERE handle = ?", + [self.serializer.object_to_string(obj), obj.handle], ) else: # Insert the object: self.dbapi.execute( - f"INSERT INTO {table} (handle, blob_data) VALUES (?, ?)", - [obj.handle, pickle.dumps(obj.serialize())], + f"INSERT INTO {table} (handle, {self.serializer.data_field}) VALUES (?, ?)", + [obj.handle, self.serializer.object_to_string(obj)], ) self._update_secondary_values(obj) self._update_backlinks(obj, trans) if not trans.batch: if old_data: - trans.add(obj_key, TXNUPD, obj.handle, old_data, obj.serialize()) + trans.add(obj_key, TXNUPD, obj.handle, old_data, to_dict(obj)) else: - trans.add(obj_key, TXNADD, obj.handle, None, obj.serialize()) + trans.add(obj_key, TXNADD, obj.handle, None, to_dict(obj)) return old_data @@ -660,19 +696,19 @@ def _commit_raw(self, data, obj_key): changes as part of the transaction. """ table = KEY_TO_NAME_MAP[obj_key] - handle = data[0] + handle = data["handle"] if self._has_handle(obj_key, handle): # update the object: self.dbapi.execute( - f"UPDATE {table} SET blob_data = ? WHERE handle = ?", - [pickle.dumps(data), handle], + f"UPDATE {table} SET {self.serializer.data_field} = ? WHERE handle = ?", + [self.serializer.data_to_string(data), handle], ) else: # Insert the object: self.dbapi.execute( - f"INSERT INTO {table} (handle, blob_data) VALUES (?, ?)", - [handle, pickle.dumps(data)], + f"INSERT INTO {table} (handle, {self.serializer.data_field}) VALUES (?, ?)", + [handle, self.serializer.data_to_string(data)], ) def _update_backlinks(self, obj, transaction): @@ -829,11 +865,11 @@ def _iter_raw_data(self, obj_key): """ table = KEY_TO_NAME_MAP[obj_key] with self.dbapi.cursor() as cursor: - cursor.execute(f"SELECT handle, blob_data FROM {table}") + cursor.execute(f"SELECT handle, {self.serializer.data_field} FROM {table}") rows = cursor.fetchmany() while rows: for row in rows: - yield (row[0], pickle.loads(row[1])) + yield (row[0], self.serializer.string_to_data(row[1])) rows = cursor.fetchmany() def _iter_raw_place_tree_data(self): @@ -844,12 +880,13 @@ def _iter_raw_place_tree_data(self): while to_do: handle = to_do.pop() self.dbapi.execute( - "SELECT handle, blob_data FROM place WHERE enclosed_by = ?", [handle] + f"SELECT handle, {self.serializer.data_field} FROM place WHERE enclosed_by = ?", + [handle], ) rows = self.dbapi.fetchall() for row in rows: to_do.append(row[0]) - yield (row[0], pickle.loads(row[1])) + yield (row[0], self.serializer.string_to_data(row[1])) def reindex_reference_map(self, callback): """ @@ -891,7 +928,7 @@ def reindex_reference_map(self, callback): logging.info("Rebuilding %s reference map", class_func.__name__) with cursor_func() as cursor: for _, val in cursor: - obj = class_func.create(val) + obj = self.serializer.data_to_object(class_func, val) references = set(obj.get_referenced_handles_recursively()) # handle addition of new references for ref_class_name, ref_handle in references: @@ -974,20 +1011,24 @@ def _get_gramps_ids(self, obj_key): def _get_raw_data(self, obj_key, handle): table = KEY_TO_NAME_MAP[obj_key] - self.dbapi.execute(f"SELECT blob_data FROM {table} WHERE handle = ?", [handle]) + self.dbapi.execute( + f"SELECT {self.serializer.data_field} FROM {table} WHERE handle = ?", + [handle], + ) row = self.dbapi.fetchone() if row: - return pickle.loads(row[0]) + return self.serializer.string_to_data(row[0]) return None def _get_raw_from_id_data(self, obj_key, gramps_id): table = KEY_TO_NAME_MAP[obj_key] self.dbapi.execute( - f"SELECT blob_data FROM {table} WHERE gramps_id = ?", [gramps_id] + f"SELECT {self.serializer.data_field} FROM {table} WHERE gramps_id = ?", + [gramps_id], ) row = self.dbapi.fetchone() if row: - return pickle.loads(row[0]) + return self.serializer.string_to_data(row[0]) return None def get_gender_stats(self): @@ -1042,15 +1083,15 @@ def undo_data(self, data, handle, obj_key): else: if self._has_handle(obj_key, handle): self.dbapi.execute( - f"UPDATE {table} SET blob_data = ? WHERE handle = ?", - [pickle.dumps(data), handle], + f"UPDATE {table} SET {self.serializer.data_field} = ? WHERE handle = ?", + [self.serializer.data_to_string(data), handle], ) else: self.dbapi.execute( - f"INSERT INTO {table} (handle, blob_data) VALUES (?, ?)", - [handle, pickle.dumps(data)], + f"INSERT INTO {table} (handle, {self.serializer.data_field}) VALUES (?, ?)", + [handle, self.serializer.data_to_string(data)], ) - obj = self._get_table_func(cls)["class_func"].create(data) + obj = from_dict(data) self._update_secondary_values(obj) def get_surname_list(self): diff --git a/gramps/plugins/db/dbapi/sqlite.py b/gramps/plugins/db/dbapi/sqlite.py index aec471b745c..e27a00284c3 100644 --- a/gramps/plugins/db/dbapi/sqlite.py +++ b/gramps/plugins/db/dbapi/sqlite.py @@ -191,6 +191,24 @@ def table_exists(self, table): ) return self.fetchone()[0] != 0 + def column_exists(self, table, column): + """ + Test whether the specified SQL column exists in the specified table. + + :param table: table name to check. + :type table: str + :param column: column name to check. + :type column: str + :returns: True if the column exists, False otherwise. + :rtype: bool + """ + self.execute( + "SELECT COUNT(*) " + f"FROM pragma_table_info('{table}') " + f"WHERE name = '{column}'" + ) + return self.fetchone()[0] != 0 + def close(self): """ Close the current database. diff --git a/gramps/plugins/gramplet/sessionloggramplet.py b/gramps/plugins/gramplet/sessionloggramplet.py index c6a4edf8222..57d3defb3fe 100644 --- a/gramps/plugins/gramplet/sessionloggramplet.py +++ b/gramps/plugins/gramplet/sessionloggramplet.py @@ -1,6 +1,6 @@ # Gramps - a GTK+/GNOME based genealogy program # -# Copyright (C) 2007-2009 Douglas S. Blank +# Copyright (C) 2007-2009,2024 Douglas S. Blank # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -30,6 +30,7 @@ # ------------------------------------------------------------------------ from gramps.gen.lib import Person, Family +from gramps.gen.lib.serialize import from_dict from gramps.gen.db import PERSON_KEY, FAMILY_KEY, TXNDEL from gramps.gen.plug import Gramplet from gramps.gen.display.name import displayer as name_displayer @@ -128,8 +129,7 @@ def log(self, ltype, action, handles): and trans_type == TXNDEL and hndl == handle ): - person = Person() - person.unserialize(old_data) + person = from_dict(old_data) name = name_displayer.display(person) break elif ltype == "Family": @@ -150,8 +150,7 @@ def log(self, ltype, action, handles): and trans_type == TXNDEL and hndl == handle ): - family = Family() - family.unserialize(old_data) + family = from_dict(old_data) name = family_name(family, self.dbstate.db, name) break self.append_text(name) diff --git a/gramps/plugins/importer/importvcard.py b/gramps/plugins/importer/importvcard.py index cd1cd424447..1d45095e9cb 100644 --- a/gramps/plugins/importer/importvcard.py +++ b/gramps/plugins/importer/importvcard.py @@ -525,11 +525,11 @@ def add_street(strng): else: addr.set_street(strng) - addr.add_street = add_street + addr._add_street = add_street set_func = [ - "add_street", - "add_street", - "add_street", + "_add_street", + "_add_street", + "_add_street", "set_city", "set_state", "set_postal_code", diff --git a/gramps/plugins/importer/importxml.py b/gramps/plugins/importer/importxml.py index b6926e94836..8e047c68391 100644 --- a/gramps/plugins/importer/importxml.py +++ b/gramps/plugins/importer/importxml.py @@ -3,7 +3,7 @@ # # Copyright (C) 2000-2007 Donald N. Allingham # Copyright (C) 200?-2013 Benny Malengier -# Copyright (C) 2009 Douglas S. Blank +# Copyright (C) 2009,2024 Douglas S. Blank # Copyright (C) 2010-2011 Nick Hall # Copyright (C) 2011 Michiel D. Nauta # Copyright (C) 2011 Tim G L Lyons @@ -91,6 +91,7 @@ Tag, Url, ) +from gramps.gen.lib.serialize import from_dict from gramps.gen.db import DbTxn # from gramps.gen.db.write import CLASS_TO_KEY_MAP @@ -839,7 +840,8 @@ class object of a primary object. "tag": self.db.get_raw_tag_data, }[target] raw = get_raw_obj_data(handle) - prim_obj.unserialize(raw) + temp_obj = from_dict(raw) + prim_obj.set_object_state(temp_obj.get_object_state()) self.import_handles[orig_handle][target][INSTANTIATED] = True return handle elif handle in self.import_handles: @@ -1000,7 +1002,8 @@ def inaugurate_id(self, id_, key, prim_obj): handle = id2handle_map.get(gramps_id) if handle: raw = get_raw_obj_data(handle) - prim_obj.unserialize(raw) + temp_obj = from_dict(raw) + prim_obj.set_object_state(temp_obj.get_object_state()) else: handle = create_id() while has_handle_func(handle): diff --git a/gramps/plugins/lib/libgedcom.py b/gramps/plugins/lib/libgedcom.py index ba67d07435a..5bead8826db 100644 --- a/gramps/plugins/lib/libgedcom.py +++ b/gramps/plugins/lib/libgedcom.py @@ -6,6 +6,7 @@ # Copyright (C) 2010 Nick Hall # Copyright (C) 2011 Tim G L Lyons # Copyright (C) 2016 Paul R. Culley +# Copyright (C) 2024 Doug Blank # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -157,6 +158,7 @@ PlaceRef, PlaceName, ) +from gramps.gen.lib.serialize import from_dict, to_dict from gramps.gen.db import DbTxn from gramps.gen.updatecallback import UpdateCallback from gramps.gen.utils.file import media_path @@ -2163,7 +2165,7 @@ class GedcomParser(UpdateCallback): __TRUNC_MSG = _( "Your GEDCOM file is corrupted. " "It appears to have been truncated." ) - _EMPTY_LOC = Location().serialize() + _EMPTY_LOC = to_dict(Location()) SyntaxError = "Syntax Error" BadFile = "Not a GEDCOM file" @@ -3185,11 +3187,11 @@ def __find_or_create_person(self, gramps_id): already used (is in the db), we return the item in the db. Otherwise, we create a new person, assign the handle and Gramps ID. """ - person = Person() intid = self.gid2id.get(gramps_id) if self.dbase.has_person_handle(intid): - person.unserialize(self.dbase.get_raw_person_data(intid)) + person = from_dict(self.dbase.get_raw_person_data(intid)) else: + person = Person() intid = self.__find_from_handle(gramps_id, self.gid2id) person.set_handle(intid) person.set_gramps_id(gramps_id) @@ -3201,16 +3203,16 @@ def __find_or_create_family(self, gramps_id): already used (is in the db), we return the item in the db. Otherwise, we create a new family, assign the handle and Gramps ID. """ - family = Family() - # Add a counter for reordering the children later: - family.child_ref_count = 0 intid = self.fid2id.get(gramps_id) if self.dbase.has_family_handle(intid): - family.unserialize(self.dbase.get_raw_family_data(intid)) + family = from_dict(self.dbase.get_raw_family_data(intid)) else: + family = Family() intid = self.__find_from_handle(gramps_id, self.fid2id) family.set_handle(intid) family.set_gramps_id(gramps_id) + # Add a counter for reordering the children later: + family._child_ref_count = 0 return family def __find_or_create_media(self, gramps_id): @@ -3219,11 +3221,11 @@ def __find_or_create_media(self, gramps_id): already used (is in the db), we return the item in the db. Otherwise, we create a new media object, assign the handle and Gramps ID. """ - obj = Media() intid = self.oid2id.get(gramps_id) if self.dbase.has_media_handle(intid): - obj.unserialize(self.dbase.get_raw_media_data(intid)) + obj = from_dict(self.dbase.get_raw_media_data(intid)) else: + obj = Media() intid = self.__find_from_handle(gramps_id, self.oid2id) obj.set_handle(intid) obj.set_gramps_id(gramps_id) @@ -3237,11 +3239,11 @@ def __find_or_create_source(self, gramps_id): db. Otherwise, we create a new source, assign the handle and Gramps ID. """ - obj = Source() intid = self.sid2id.get(gramps_id) if self.dbase.has_source_handle(intid): - obj.unserialize(self.dbase.get_raw_source_data(intid)) + obj = from_dict(self.dbase.get_raw_source_data(intid)) else: + obj = Source() intid = self.__find_from_handle(gramps_id, self.sid2id) obj.set_handle(intid) obj.set_gramps_id(gramps_id) @@ -3256,11 +3258,11 @@ def __find_or_create_repository(self, gramps_id): Some GEDCOM "flavors" destroy the specification, and declare the repository inline instead of in a object. """ - repository = Repository() intid = self.rid2id.get(gramps_id) if self.dbase.has_repository_handle(intid): - repository.unserialize(self.dbase.get_raw_repository_data(intid)) + repository = from_dict(self.dbase.get_raw_repository_data(intid)) else: + repository = Repository() intid = self.__find_from_handle(gramps_id, self.rid2id) repository.set_handle(intid) repository.set_gramps_id(gramps_id) @@ -3274,7 +3276,6 @@ def __find_or_create_note(self, gramps_id): If no Gramps ID is passed in, we not only make a Note with GID, we commit it. """ - note = Note() if not gramps_id: need_commit = True gramps_id = self.dbase.find_next_note_gramps_id() @@ -3283,8 +3284,9 @@ def __find_or_create_note(self, gramps_id): intid = self.nid2id.get(gramps_id) if self.dbase.has_note_handle(intid): - note.unserialize(self.dbase.get_raw_note_data(intid)) + note = from_dict(self.dbase.get_raw_note_data(intid)) else: + note = Note() intid = self.__find_from_handle(gramps_id, self.nid2id) note.set_handle(intid) note.set_gramps_id(gramps_id) @@ -3302,7 +3304,7 @@ def __loc_is_empty(self, location): """ if location is None: return True - elif location.serialize() == self._EMPTY_LOC: + elif to_dict(location) == self._EMPTY_LOC: return True elif location.is_empty(): return True @@ -5646,8 +5648,8 @@ def set_child_ref_order(self, family, child_ref): order given in the FAM section. """ family.child_ref_list.remove(child_ref) - family.child_ref_list.insert(family.child_ref_count, child_ref) - family.child_ref_count += 1 + family.child_ref_list.insert(family._child_ref_count, child_ref) + family._child_ref_count += 1 def __family_slgs(self, line, state): """ diff --git a/gramps/plugins/lib/libmixin.py b/gramps/plugins/lib/libmixin.py index b8b7e7faf8d..7a4113bcefa 100644 --- a/gramps/plugins/lib/libmixin.py +++ b/gramps/plugins/lib/libmixin.py @@ -2,6 +2,7 @@ # Gramps - a GTK+/GNOME based genealogy program # # Copyright (C) 2000-2007 Donald N. Allingham +# Copyright (C) 2024 Doug Blank # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -38,6 +39,7 @@ Note, Tag, ) +from gramps.gen.lib.serialize import from_dict # ------------------------------------------------------------------------------ @@ -77,7 +79,7 @@ def __find_primary_from_handle( new = True raw = get_raw_obj_data(handle) if raw is not None: - obj.unserialize(raw) + obj = from_dict(raw) # references create object with id None before object is really made if obj.gramps_id is not None: new = False @@ -103,7 +105,7 @@ def __find_table_from_handle( handle = str(handle) raw = get_raw_obj_data(handle) if raw is not None: - obj.unserialize(raw) + obj = from_dict(raw) return obj, False else: obj.set_handle(handle) diff --git a/gramps/plugins/test/imports_test.py b/gramps/plugins/test/imports_test.py index ffbbca7f94a..f4200c7b148 100644 --- a/gramps/plugins/test/imports_test.py +++ b/gramps/plugins/test/imports_test.py @@ -36,7 +36,7 @@ config.set("preferences.date-format", 0) from gramps.gen.db.utils import import_as_dict -from gramps.gen.merge.diff import diff_dbs, to_struct +from gramps.gen.merge.diff import diff_dbs, to_dict from gramps.gen.simple import SimpleAccess from gramps.gen.utils.id import set_det_id from gramps.gen.user import User @@ -89,7 +89,7 @@ def prepare_result(self, diffs, added, missing): if diffs: for diff in diffs: obj_type, item1, item2 = diff - msg = self._report_diff(obj_type, to_struct(item1), to_struct(item2)) + msg = self._report_diff(obj_type, to_dict(item1), to_dict(item2)) if msg != "": if hasattr(item1, "gramps_id"): self.msg += "%s: %s handle=%s\n" % ( diff --git a/gramps/plugins/test/tools_test.py b/gramps/plugins/test/tools_test.py index efccb8346fa..3ba03c84437 100644 --- a/gramps/plugins/test/tools_test.py +++ b/gramps/plugins/test/tools_test.py @@ -149,11 +149,9 @@ def test_tcg_and_check_and_repair(self): self.assertTrue(check_res(out, err, expect, do_out=True)) out, err = call("-O", TREE_NAME, "-y", "-a", "tool", "-p", "name=check") expect = [ - "7 broken child/family links were fixed", - "4 broken spouse/family links were fixed", "1 place alternate name fixed", - "10 media objects were referenced, but not found", - "References to 10 missing media objects were kept", + "9 media objects were referenced, but not found", + "References to 9 missing media objects were kept", "3 events were referenced, but not found", "1 invalid birth event name was fixed", "1 invalid death event name was fixed", @@ -161,13 +159,13 @@ def test_tcg_and_check_and_repair(self): "16 citations were referenced, but not found", "19 sources were referenced, but not found", "9 Duplicated Gramps IDs fixed", - "7 empty objects removed", + "9 empty objects removed:", "1 person objects", "1 family objects", "1 event objects", "1 source objects", - "0 media objects", - "0 place objects", + "1 media objects", + "1 place objects", "1 repository objects", "1 note objects", ] diff --git a/gramps/plugins/tool/check.py b/gramps/plugins/tool/check.py index d5ac2fc6a0c..8d948c91822 100644 --- a/gramps/plugins/tool/check.py +++ b/gramps/plugins/tool/check.py @@ -6,6 +6,7 @@ # Copyright (C) 2010 Jakim Friant # Copyright (C) 2011 Tim G L Lyons # Copyright (C) 2012 Michiel D. Nauta +# Copyright (C) 2024 Doug Blank # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -75,6 +76,7 @@ StyledTextTagType, Tag, ) +from gramps.gen.lib.serialize import to_dict from gramps.gen.db import DbTxn, CLASS_TO_KEY_MAP from gramps.gen.config import config from gramps.gen.utils.id import create_id @@ -423,16 +425,16 @@ def fix_encoding(self): error_count = 0 for handle in self.db.get_media_handles(): data = self.db.get_raw_media_data(handle) - if not isinstance(data[2], str) or not isinstance(data[4], str): + if not isinstance(data["path"], str) or not isinstance(data["desc"], str): obj = self.db.get_media_from_handle(handle) - if not isinstance(data[2], str): + if not isinstance(data["path"], str): obj.path = obj.path.decode("utf-8") logging.warning( " FAIL: encoding error on media object " '"%(gid)s" path "%(path)s"', {"gid": obj.gramps_id, "path": obj.path}, ) - if not isinstance(data[4], str): + if not isinstance(data["desc"], str): obj.desc = obj.desc.decode("utf-8") logging.warning( " FAIL: encoding error on media object " @@ -442,11 +444,11 @@ def fix_encoding(self): self.db.commit_media(obj, self.trans) error_count += 1 # Once we are here, fix the mime string if not str - if not isinstance(data[3], str): + if not isinstance(data["mime"], str): obj = self.db.get_media_from_handle(handle) try: - if data[3] == str(data[3]): - obj.mime = str(data[3]) + if data["mime"] == str(data["mime"]): + obj.mime = str(data["mime"]) else: obj.mime = "" except: @@ -907,34 +909,23 @@ def fs_ok_clicked(obj): logging.info(" OK: no missing photos found") def cleanup_empty_objects(self): - # the position of the change column in the primary objects - CHANGE_PERSON = 17 - CHANGE_FAMILY = 12 - CHANGE_EVENT = 10 - CHANGE_SOURCE = 8 - CHANGE_CITATION = 9 - CHANGE_PLACE = 11 - CHANGE_MEDIA = 8 - CHANGE_REPOS = 7 - CHANGE_NOTE = 5 - - empty_person_data = Person().serialize() - empty_family_data = Family().serialize() - empty_event_data = Event().serialize() - empty_source_data = Source().serialize() - empty_citation_data = Citation().serialize() - empty_place_data = Place().serialize() - empty_media_data = Media().serialize() - empty_repos_data = Repository().serialize() - empty_note_data = Note().serialize() + empty_person_data = to_dict(Person()) + empty_family_data = to_dict(Family()) + empty_event_data = to_dict(Event()) + empty_source_data = to_dict(Source()) + empty_citation_data = to_dict(Citation()) + empty_place_data = to_dict(Place()) + empty_media_data = to_dict(Media()) + empty_repos_data = to_dict(Repository()) + empty_note_data = to_dict(Note()) _db = self.db - def _empty(empty, flag): + def _empty(empty): """Closure for dispatch table, below""" def _fx(value): - return self._check_empty(value, empty, flag) + return self._check_empty(value, empty) return _fx @@ -954,7 +945,7 @@ def _fx(value): _db.get_person_cursor, _db.get_number_of_people, _("Looking for empty people records"), - _empty(empty_person_data, CHANGE_PERSON), + _empty(empty_person_data), _db.remove_person, ), ( @@ -963,7 +954,7 @@ def _fx(value): _db.get_family_cursor, _db.get_number_of_families, _("Looking for empty family records"), - _empty(empty_family_data, CHANGE_FAMILY), + _empty(empty_family_data), _db.remove_family, ), ( @@ -972,7 +963,7 @@ def _fx(value): _db.get_event_cursor, _db.get_number_of_events, _("Looking for empty event records"), - _empty(empty_event_data, CHANGE_EVENT), + _empty(empty_event_data), _db.remove_event, ), ( @@ -981,7 +972,7 @@ def _fx(value): _db.get_source_cursor, _db.get_number_of_sources, _("Looking for empty source records"), - _empty(empty_source_data, CHANGE_SOURCE), + _empty(empty_source_data), _db.remove_source, ), ( @@ -990,7 +981,7 @@ def _fx(value): _db.get_citation_cursor, _db.get_number_of_citations, _("Looking for empty citation records"), - _empty(empty_citation_data, CHANGE_CITATION), + _empty(empty_citation_data), _db.remove_citation, ), ( @@ -999,7 +990,7 @@ def _fx(value): _db.get_place_cursor, _db.get_number_of_places, _("Looking for empty place records"), - _empty(empty_place_data, CHANGE_PLACE), + _empty(empty_place_data), _db.remove_place, ), ( @@ -1008,7 +999,7 @@ def _fx(value): _db.get_media_cursor, _db.get_number_of_media, _("Looking for empty media records"), - _empty(empty_media_data, CHANGE_MEDIA), + _empty(empty_media_data), _db.remove_media, ), ( @@ -1017,7 +1008,7 @@ def _fx(value): _db.get_repository_cursor, _db.get_number_of_repositories, _("Looking for empty repository records"), - _empty(empty_repos_data, CHANGE_REPOS), + _empty(empty_repos_data), _db.remove_repository, ), ( @@ -1026,7 +1017,7 @@ def _fx(value): _db.get_note_cursor, _db.get_number_of_notes, _("Looking for empty note records"), - _empty(empty_note_data, CHANGE_NOTE), + _empty(empty_note_data), _db.remove_note, ), ) @@ -1065,16 +1056,15 @@ def _fx(value): if len(self.empty_objects[the_type]) == 0: logging.info(" OK: no empty %s found", the_type) - def _check_empty(self, data, empty_data, changepos): + def _check_empty(self, data, empty_data): """compare the data with the data of an empty object change, handle and gramps_id are not compared""" - if changepos is not None: - return ( - data[2:changepos] == empty_data[2:changepos] - and data[changepos + 1 :] == empty_data[changepos + 1 :] - ) - else: - return data[2:] == empty_data[2:] + for key in empty_data: + if key in ["change", "gramps_id", "handle"]: + continue + if data[key] != empty_data[key]: + return False + return True def cleanup_empty_families(self, dummy): fhandle_list = self.db.get_family_handles() diff --git a/gramps/plugins/tool/mediamanager.py b/gramps/plugins/tool/mediamanager.py index 16d855ef9dc..628f1248344 100644 --- a/gramps/plugins/tool/mediamanager.py +++ b/gramps/plugins/tool/mediamanager.py @@ -6,6 +6,7 @@ # Copyright (C) 2008 Brian G. Matherly # Copyright (C) 2010 Jakim Friant # Copyright (C) 2012 Nick Hall +# Copyright (C) 2024 Doug Blank # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -50,6 +51,7 @@ from gramps.gen.const import URL_MANUAL_PAGE, ICON, SPLASH from gramps.gui.display import display_help from gramps.gen.lib import Media +from gramps.gen.lib.serialize import from_dict from gramps.gen.db import DbTxn from gramps.gen.updatecallback import UpdateCallback from gramps.gui.plug import tool @@ -568,8 +570,7 @@ def _prepare(self): self.set_total(self.db.get_number_of_media()) with self.db.get_media_cursor() as cursor: for handle, data in cursor: - obj = Media() - obj.unserialize(data) + obj = from_dict(data) if obj.get_path().find(from_text) != -1: self.handle_list.append(handle) self.path_list.append(obj.path) @@ -608,8 +609,7 @@ def _prepare(self): self.set_total(self.db.get_number_of_media()) with self.db.get_media_cursor() as cursor: for handle, data in cursor: - obj = Media() - obj.unserialize(data) + obj = from_dict(data) if not os.path.isabs(obj.path): self.handle_list.append(handle) self.path_list.append(obj.path) @@ -647,8 +647,7 @@ def _prepare(self): self.set_total(self.db.get_number_of_media()) with self.db.get_media_cursor() as cursor: for handle, data in cursor: - obj = Media() - obj.unserialize(data) + obj = from_dict(data) if os.path.isabs(obj.path): self.handle_list.append(handle) self.path_list.append(obj.path) @@ -689,8 +688,7 @@ def _prepare(self): self.set_total(self.db.get_number_of_media()) with self.db.get_media_cursor() as cursor: for handle, data in cursor: - obj = Media() - obj.unserialize(data) + obj = from_dict(data) self.handle_list.append(handle) full_path = media_path_full(self.db, obj.path) self.path_list.append(full_path) diff --git a/gramps/plugins/tool/rebuildgenderstat.py b/gramps/plugins/tool/rebuildgenderstat.py index 9b46fb2cbeb..84d3d8bb551 100644 --- a/gramps/plugins/tool/rebuildgenderstat.py +++ b/gramps/plugins/tool/rebuildgenderstat.py @@ -2,6 +2,7 @@ # Gramps - a GTK+/GNOME based genealogy program # # Copyright (C) 2012 Benny Malengier +# Copyright (C) 2024 Doug Blank # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -55,6 +56,7 @@ from gramps.gui.dialog import OkDialog from gramps.gen.updatecallback import UpdateCallback from gramps.gen.lib import Name +from gramps.gen.lib.serialize import from_dict # ------------------------------------------------------------------------- # @@ -62,10 +64,6 @@ # # ------------------------------------------------------------------------- -COLUMN_GENDER = 2 -COLUMN_NAME = 3 -COLUMN_ALTNAMES = 4 - class RebuildGenderStat(tool.Tool, UpdateCallback): def __init__(self, dbstate, user, options_class, name, callback=None): @@ -114,13 +112,13 @@ def rebuild_genderstats(self): # loop over database and store the sort field, and the handle, and # allow for a third iter for key, data in cursor: - rawprimname = data[COLUMN_NAME] - rawaltnames = data[COLUMN_ALTNAMES] - primary_name = Name().unserialize(rawprimname).get_first_name() + rawprimname = data["primary_name"] + rawaltnames = data["alternate_names"] + primary_name = from_dict(rawprimname).get_first_name() alternate_names = [ - Name().unserialize(name).get_first_name() for name in rawaltnames + from_dict(name).get_first_name() for name in rawaltnames ] - self.db.genderStats.count_name(primary_name, data[COLUMN_GENDER]) + self.db.genderStats.count_name(primary_name, data["gender"]) # ------------------------------------------------------------------------ diff --git a/test/blob_to_json_test.py b/test/blob_to_json_test.py new file mode 100644 index 00000000000..766657bda5d --- /dev/null +++ b/test/blob_to_json_test.py @@ -0,0 +1,41 @@ +from gramps.gen.db.utils import open_database +from gramps.gen.lib.serialize import to_dict, from_dict +from gramps.gen.db.conversion_tools import convert_21 + +# 1. Prepare: create a database named "Example" in version 20 +# containing the blob data + +# 2. Open the database in latest Gramps to convert the database +# to version 21 + +db = open_database("Example") + +# This is a version 21 database + +# But we tell it to use the blob data: + +db.set_serializer("blob") + +for table_name in db._get_table_func(): + print("Testing %s..." % table_name) + get_array_from_handle = db._get_table_func(table_name, "raw_func") + iter_objects = db._get_table_func(table_name, "iter_func") + + for obj in iter_objects(): + # We convert the object into the JSON dicts: + json_data = to_dict(obj) + + # We get the blob array: + array = get_array_from_handle(obj.handle) + + # We convert the array to JSON dict using the + # conversion code to convert array directly to + # dict + + convert_data = convert_21(table_name, array) + + # Now we make sure they are identical in types + # and values: + assert convert_data == json_data + +db.close()