xpoJson2rdf.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "e090c393",
   "metadata": {},
   "source": [
    "## XPO JSON to RDF\n",
    " * Converts the JSON representation of the XPO knowledge graph to RDF\n",
    "   * The version of the XPO JSON file is from this [GitHub repository](https://github.com/e-spaulding/xpo)\n",
    "   * a 2023 paper describing it is [here](https://github.com/Ebiquity/xpo_rdf/blob/main/ISA2023_DWD_Overlay_Workshop_Paper.pdf)\n",
    " * The resulting RDF graph uses two name space prefixes defined in purl that redirect to the xpo.nt file in the repository on GitHub\n",
    "   * xpo: http://purl.org/xpo/\n",
    "   * dwd: http://purl.org/dwd/\n",
    " * We tried using a simpler approach that would make minor changes to the JSON file to make it JSON-LD and then using a standard tool to covert it to an RDF representation, but the format of the original xpo JSON file required to many structural changes.\n",
    "\n",
    " * we might want to extend this to add an addional schema statements to allow more inferences to be drawn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "8b172b9b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from rdflib import Graph, URIRef, Namespace, BNode, Literal\n",
    "from rdflib.namespace import RDF, OWL, RDFS\n",
    "import shortuuid \n",
    "import json"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "42c75e45",
   "metadata": {},
   "source": [
    "### Some helper functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "cdfe6391",
   "metadata": {},
   "outputs": [],
   "source": [
    "def bnode(prefix = ''):\n",
    "    # custom BNode-like function adds a prefix to a short uuid sequence\n",
    "    if prefix:\n",
    "        return BNode(prefix + '_' + shortuuid.uuid()[:5])\n",
    "    else:\n",
    "        return BNode(shortuuid.uuid()[:5])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0937dfeb",
   "metadata": {},
   "source": [
    "## Create and inital RDF graph and bind some namespace prefixes in it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "24da5917",
   "metadata": {},
   "outputs": [],
   "source": [
    "GRAPH = Graph()\n",
    "\n",
    "# unsure if we should use xpo or dwd as the profix\n",
    "XPO = Namespace('http://purl.org/xpo/')\n",
    "DWD = Namespace('http://purl.org/dwd/')\n",
    "\n",
    "# use this namespace for the ontology\n",
    "ONT = XPO\n",
    "\n",
    "WD = Namespace('http://www.wikidata.org/wiki/')\n",
    "WDP = Namespace('http://www.wikidata.org/wiki/Property:')\n",
    "\n",
    "GRAPH.bind(\"dwd\", DWD)\n",
    "GRAPH.bind(\"xpo\", XPO)\n",
    "\n",
    "GRAPH.bind(\"owl\", OWL)\n",
    "GRAPH.bind(\"rdf\", RDF)\n",
    "GRAPH.bind('rdfs', RDFS)\n",
    "GRAPH.bind('wd', WD)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "97ec00b4",
   "metadata": {},
   "source": [
    "### Get the XPO JSON file and extract the four subsets of data\n",
    " * each is a dictionary where the keys are node ids and the values are dictionaries of their properties and values.\n",
    " * for some prperaties the value is a list -- for these with generate multiple edges, one for each value in the list.\n",
    " * we start by extracing the four top-level categories: events, entities, relations, and temopral_relations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "7fb0718a",
   "metadata": {},
   "outputs": [],
   "source": [
    "xpo_data = json.load(open('xpo.json'))\n",
    "event = xpo_data['events']\n",
    "entity = xpo_data['entities']\n",
    "relation = xpo_data['relations']\n",
    "temporal_relation = xpo_data['temporal_relations']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ed9a3af7",
   "metadata": {},
   "source": [
    "### This is a list of all of the properties found in version 5.4.7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "fec148b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# these were extracted from the json file\n",
    "all_properties = ['arguments', 'comment', 'constraints', 'curated_by', \n",
    "                  'dwd_arg_name', 'entities', 'events', 'ldc_argument_output_value', \n",
    "                  'ldc_arguments', 'ldc_code', 'ldc_constraints', 'ldc_name', 'ldc_types', \n",
    "                  'mapping_types', 'name', 'other_pb_rolesets', 'overlay_parents', \n",
    "                  'pb_mapping', 'pb_roleset', 'related_qnodes', 'relations', 'short_name', \n",
    "                  'similar_nodes', 'similarity_type', 'template', 'template_curation', \n",
    "                  'temporal_relations', 'type', 'version', 'wd_description', 'wd_node', \n",
    "                  'wd_slot']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "842dcc7d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_generic(data, DWD=XPO, type=\"?\", stop=0):\n",
    "    count = 0 # just used for testing\n",
    "    for (node, edges) in data.items():\n",
    "        count += 1\n",
    "        if stop > 0 and count > stop:\n",
    "            break\n",
    "        node = URIRef(DWD+node)\n",
    "        for (property, value) in edges.items():\n",
    "            if property in ['type', 'comment', 'curated_by', 'description', \n",
    "                              'wd_node', 'name', 'wd_description', 'template', \n",
    "                              'template_curation', 'pb_roleset']:\n",
    "                # properties with a string value\n",
    "                GRAPH.add((node, URIRef(DWD + property), Literal(value)))\n",
    "            elif property == \"overlay_parents\":\n",
    "                if isinstance(value, list):\n",
    "                    for v in value:\n",
    "                        overlay_node = bnode('OVERLAY')\n",
    "                        GRAPH.add((node, DWD.overlay, overlay_node)) \n",
    "                        #GRAPH.add((overlay_node, RDF.type, DWD.overlay)) #overlay?\n",
    "                        GRAPH.add((overlay_node, DWD.overlay_parent, URIRef(WD+v['wd_node'])))\n",
    "                        GRAPH.add((overlay_node, DWD.overlay_parent_name, Literal(v['name'])))\n",
    "                else:\n",
    "                    print(f\"Bad property-values {property} {value}\")\n",
    "            elif property == 'similar_nodes':\n",
    "                if isinstance(value, list):\n",
    "                    for v in value:\n",
    "                        similar_node = bnode('SIMILAR')\n",
    "                        GRAPH.add((node, DWD.similarNode, similar_node)) \n",
    "                        # GRAPH.add((similar_node, RDF.type, DWD.similar_node))\n",
    "                        GRAPH.add((similar_node, DWD.wd_node, URIRef(WD+v['wd_node'])))\n",
    "                        GRAPH.add((similar_node, DWD.name, Literal(v['name'])))\n",
    "                        GRAPH.add((similar_node, DWD.similarity_type, URIRef(DWD+v['similarity_type'])))\n",
    "                else:\n",
    "                    print(f\"Bad property-values {property} {value}\")\n",
    "            elif property == 'ldc_types':\n",
    "                if isinstance(value, list):\n",
    "                    # should be a list of dicts\n",
    "                    for v in value:\n",
    "                        ldc_type_node = bnode('LDCTYPE')\n",
    "                        GRAPH.add((node, DWD.ldc_type, ldc_type_node))\n",
    "                        for (vname, vvalue) in v.items():\n",
    "                            if vname == 'name':\n",
    "                                GRAPH.add((ldc_type_node, DWD.name, Literal(vvalue)))\n",
    "                            elif vname == 'ldc_code':\n",
    "                                GRAPH.add((ldc_type_node, DWD.ldc_code, Literal(vvalue)))\n",
    "                            elif vname == 'other_pb_rolesets':\n",
    "                                for pb_roleset in vvalue:\n",
    "                                    GRAPH.add((ldc_type_node, DWD.other_pb_roleset, Literal(pb_roleset)))\n",
    "                            elif vname == 'ldc_arguments':\n",
    "                                for ldc_arg in vvalue:\n",
    "                                    ldc_arg_node = bnode('LDCARG')\n",
    "                                    GRAPH.add((ldc_type_node, DWD.ldc_argument, ldc_arg_node))\n",
    "                                    for (ldc_arg_name, ldc_arg_value) in ldc_arg.items():\n",
    "                                        if ldc_arg_name in [\"ldc_name\", \"ldc_argument_output_value\",\"dwd_arg_name\"]:\n",
    "                                            # all have simple string values\n",
    "                                            GRAPH.add((ldc_arg_node, DWD.ldc_code, Literal(ldc_arg_value)))\n",
    "                                    if ldc_arg_name == \"ldc_contraints\":\n",
    "                                        for ent_type in ldc_arg_value:\n",
    "                                            graph.add((ldc_arg_node, DWD.ldc_constraint, Literal(ent_type)))\n",
    "                            else:\n",
    "                                print(f\"Bad LDC_types property (unrecognized): {node} {property} {vname}\")\n",
    "                else:\n",
    "                    print(f\"Bad property-values {property} {value} (not a list)\")\n",
    "            elif property == \"arguments\":\n",
    "                if isinstance(value, list):\n",
    "                    for arg in value:\n",
    "                        # an arg should have a name, short_name and constraints\n",
    "                        arg_node = bnode('ARG')\n",
    "                        GRAPH.add((node, DWD.argument, arg_node))\n",
    "                        if 'name' in arg: GRAPH.add((arg_node, DWD.name, Literal(arg['name'])))\n",
    "                        if 'short_name' in arg: GRAPH.add((arg_node, DWD.short_name, Literal(arg['short_name'])))\n",
    "                        for arg_constraint in arg['constraints']:\n",
    "                            const_node = bnode('CONSTRAINT')\n",
    "                            GRAPH.add((arg_node, DWD.constraint, const_node))\n",
    "                            GRAPH.add((const_node, DWD.name, Literal(arg_constraint['name'])))\n",
    "                            GRAPH.add((const_node, DWD.wd_node, URIRef(WD + arg_constraint['wd_node'])))\n",
    "                else:\n",
    "                    print(f\"Bad property-values {node} {property} {value} (not a list)\")\n",
    "            elif property == \"related_qnodes\":\n",
    "                if isinstance(value, list):\n",
    "                    for v in value:\n",
    "                        wdnode = bnode('WDNODE')\n",
    "                        GRAPH.add((node, DWD.related_qnode, wdnode))\n",
    "                        GRAPH.add((wdnode, DWD.wd_node, URIRef(WD+v['wd_node'])))\n",
    "                        GRAPH.add((wdnode, DWD.name, Literal(v['name'])))\n",
    "                else:\n",
    "                    #raise exception, should be a list\n",
    "                    print(f\"Bad property-values {node} {property} {value}\")\n",
    "            # not recognized properties...\n",
    "            else:\n",
    "                if isinstance(value, list):\n",
    "                    print(f\"Unrecognized property with list of values for type {type}: {property} {value}\")\n",
    "                    if value == []:\n",
    "                        #no values for this property, so ignore\n",
    "                        pass\n",
    "                    for v in value:\n",
    "                        GRAPH.add((node, URIRef(DWD + property), Literal(v)))\n",
    "                else:\n",
    "                    print(f\"Unrecognized property for {type}: {property} {value}\")\n",
    "                    GRAPH.add((node, URIRef(DWD + property), Literal(value))) \n",
    "    return count-1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "46a911e9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 5165 event\n",
      "Found 275 entity\n",
      "Found 215 relation\n",
      "Found 7 temporal_relation\n"
     ]
    }
   ],
   "source": [
    "N = 0 # set to a small number for testing\n",
    "for (xpo_type, data) in [('event', event),\n",
    "                         ('entity', entity),\n",
    "                         ('relation', relation),\n",
    "                         ('temporal_relation', temporal_relation)]: \n",
    "    n = convert_generic(data, type=xpo_type, stop=N)\n",
    "    print(f\"Found {n} {xpo_type}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "94499496",
   "metadata": {},
   "source": [
    "### Output RDF graph to xpo.ttl and xpo.nt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "214b7f58",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Graph identifier=N90c37dd328b649c59b7ad6238e2f33d1 (<class 'rdflib.graph.Graph'>)>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "GRAPH.serialize(destination=\"xpo.ttl\", format='turtle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "404733c6",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/finin/miniforge3/envs/m1/lib/python3.9/site-packages/rdflib/plugins/serializers/nt.py:40: UserWarning: NTSerializer always uses UTF-8 encoding. Given encoding was: None\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Graph identifier=N90c37dd328b649c59b7ad6238e2f33d1 (<class 'rdflib.graph.Graph'>)>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "GRAPH.serialize(destination=\"xpo.nt\", format='nt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "950c3350",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}