adds type generation to docs ( docs included using jsdoc in types )

package.json

@@ -5,6 +5,7 @@
   "main": "index.js",
   "type": "module",
   "scripts": {
+    "generate-types": "deno run --import-map=type-generation/import_map.json --allow-write --allow-net type-generation/main.ts",
     "build": "run-script-os",
     "build:darwin:linux:default": "rm -rf docs && node index.js && cp -r static/ docs/ && cp -r beta/ docs/",
     "build:win32": "(if exist docs RMDIR /S /Q docs) && node index.js && echo d | xcopy static docs\\static && echo d | xcopy /s /q /e /y beta docs\\beta"
@@ -36,4 +37,4 @@
     "prettier": "2.3.0",
     "run-script-os": "^1.1.6"
   }
-}
+}

type-generation/.vscode/settings.json

@@ -0,0 +1,6 @@
+{
+  "deno.enable": true,
+  "deno.lint": true,
+  "deno.unstable": true,
+  "deno.importMap": "./import_map.json"
+}

type-generation/import_map.json

@@ -0,0 +1,7 @@
+{
+  "imports": {
+    "dom_parser": "https://deno.land/x/[email protected]/deno-dom-wasm.ts",
+    "progress": "https://deno.land/x/[email protected]/mod.ts",
+    "dataloader": "https://esm.sh/dataloader"
+  }
+}

type-generation/main.ts

@@ -0,0 +1,26 @@
+import { TypeGenerator } from "./type-generator/type-gen.ts";
+import { getSchemaDefinition } from "./tl-parser/tl-parser.ts";
+import { extendSchema } from "./docs/crawl.ts";
+
+if (Deno.args.length < 1) {
+  console.error("Usage: deno run main.ts <schema-url> [documentation-url]");
+  Deno.exit(1);
+}
+const documentationUrl = Deno.args[1] || "https://corefork.telegram.org/";
+const schemaUrl = Deno.args[0];
+
+console.log("getting schema");
+const schema = await fetch(schemaUrl).then((r) => r.text());
+console.log("parsing schema");
+const parseSchema = getSchemaDefinition(schema);
+console.log("fetching documentation");
+const extendedSchema = await extendSchema(parseSchema, {
+  DocsBasePath: documentationUrl,
+  language: "en",
+});
+const dtsFile = new TypeGenerator().generate(extendedSchema);
+
+Deno.writeFileSync(
+  "api.d.ts",
+  new TextEncoder().encode(dtsFile),
+);

type-generation/tl-parser/parseRow.test.ts

@@ -0,0 +1,69 @@
+import * as assert from "https://deno.land/[email protected]/testing/asserts.ts";
+import { parseRow } from "./parseRow.ts";
+
+Deno.test("should return a valid result for vector", () => {
+  const a = parseRow("vector#1cb5c415 {t:Type} # [ t ] = Vector t;");
+
+  // asserts a should be {"name":"vector","id":481674261,"generic":{"genericDescriptor":[{"key":"t","type":"Type"}],"argsOrder":["t"]},"args":[]} key by key
+  assert.assertEquals(a.name, "vector");
+  assert.assertEquals(a.id, 481674261);
+  assert.assertEquals(a.generic.genericDescriptor[0].key, "t");
+  assert.assertEquals(
+    a.generic.genericDescriptor[0].type,
+    "Type",
+  );
+  assert.assertEquals(a.generic.argsOrder![0], "t");
+  assert.assertEquals(a.type, "Vector t");
+  assert.assertEquals(a.params.length, 0);
+});
+
+Deno.test("should return a valid result for other generics", () => {
+  const a = parseRow(
+    "initConnection#c1cd5ea9 {X:Type} flags:# api_id:int device_model:string system_version:string app_version:string system_lang_code:string lang_pack:string lang_code:string proxy:flags.0?InputClientProxy params:flags.1?JSONValue query:!X = X;",
+  );
+  assert.assertEquals(a, {
+    name: "initConnection",
+    id: -1043505495,
+    generic: { genericDescriptor: [{ key: "X", type: "Type" }] },
+    params: [
+      { name: "flags", type: "#", index: 0, isOptional: false },
+      { name: "api_id", type: "int", index: 1, isOptional: false },
+      { name: "device_model", type: "string", index: 2, isOptional: false },
+      { name: "system_version", type: "string", index: 3, isOptional: false },
+      { name: "app_version", type: "string", index: 4, isOptional: false },
+      { name: "system_lang_code", type: "string", index: 5, isOptional: false },
+      { name: "lang_pack", type: "string", index: 6, isOptional: false },
+      { name: "lang_code", type: "string", index: 7, isOptional: false },
+      { name: "proxy", type: "InputClientProxy", index: 8, isOptional: true },
+      { name: "params", type: "JSONValue", index: 9, isOptional: true },
+      { name: "query", type: "X", index: 10, isOptional: false },
+    ],
+    type: "X",
+    subclassOfId: 3081909835,
+  });
+});
+
+Deno.test("should return a valid result for non-generics", () => {
+  const a = parseRow(
+    "channels.getParticipants#77ced9d0 channel:InputChannel filter:ChannelParticipantsFilter offset:int limit:int hash:long = channels.ChannelParticipants;",
+  );
+  assert.assertEquals(a, {
+    name: "channels.getParticipants",
+    id: 2010044880,
+    generic: {},
+    params: [
+      { name: "channel", type: "InputChannel", index: 0, isOptional: false },
+      {
+        name: "filter",
+        type: "ChannelParticipantsFilter",
+        index: 1,
+        isOptional: false,
+      },
+      { name: "offset", type: "int", index: 2, isOptional: false },
+      { name: "limit", type: "int", index: 3, isOptional: false },
+      { name: "hash", type: "long", index: 4, isOptional: false },
+    ],
+    type: "channels.ChannelParticipants",
+    subclassOfId: 3859443300,
+  });
+});

type-generation/tl-parser/parseRow.ts

@@ -0,0 +1,88 @@
+import { GenericParams, ParseResult } from "./types.ts";
+import { signedHexToInt } from "./utils.ts";
+import { crc32 } from "./utils.ts";
+
+const genericPattern = /(\{((\w+):(\w+),?)+\}(\s#\s\[\s?((\w+),?\s?)+\])?)/;
+const namePattern = /^([\w\.]+)#([\da-f]+)\s+/;
+const typePattern = /=.*;$/;
+
+export function parseRow(row: string): ParseResult {
+  const type = extractType(row);
+  return {
+    name: extractName(row),
+    id: extractId(row),
+    generic: extractGeneric(row),
+    params: extractParams(row),
+    type,
+    subclassOfId: crc32(type),
+  };
+}
+function extractName(row: string) {
+  try {
+    return row.match(namePattern)![1];
+  } catch (err) {
+    throw err;
+  }
+}
+function extractId(row: string) {
+  return signedHexToInt(row.match(namePattern)![2]);
+}
+
+function extractType(row: string) {
+
+    const resultOnly = row.match(typePattern)![0];
+    return resultOnly.replace(/=|;/g, "").trim();
+
+}
+
+function extractParams(row: string) {
+  const argsWithResult = removeGeneric(removeName(removeType(row)));
+  return argsWithResult
+    .split(" ")
+    .map((e) => e.trim())
+    .filter(Boolean)
+    .map((e) => e.replace(/flags.\d+/g, ""))
+    .map((e) => e.split(":"))
+    .map(([name, type], i) => ({
+      name,
+      type: type.replace(/^[\?\!]/, ""),
+      index: i,
+      isOptional: type.startsWith("?"),
+    }));
+}
+
+function extractGeneric(row: string) {
+  const remaining = removeName(row);
+  const result: Partial<GenericParams> = {};
+  if (remaining.match(genericPattern)) {
+    result.genericDescriptor = [];
+    const genericString = remaining.match(genericPattern)![1]!;
+
+    const [decleration, argsOrder] = genericString.split("#").map((e) =>
+      e.trim()
+    );
+    for (
+      const [_, key, type] of decleration.matchAll(/(\w[\w0-9]*):(\w[\w0-9]*)/g)
+    ) {
+      result.genericDescriptor.push({ key, type });
+    }
+    if (argsOrder) {
+      result.argsOrder = [...argsOrder.matchAll(/(\w[\w0-9]*)/g)].map((
+        e,
+      ) => e[0]);
+    }
+  }
+  return result as GenericParams;
+}
+
+function removeName(row: string) {
+  return row.replace(namePattern, "");
+}
+
+function removeGeneric(row: string) {
+  return row.replace(genericPattern, "");
+}
+
+function removeType(row: string) {
+  return row.replace(typePattern, "");
+}

type-generation/tl-parser/tl-parser.ts

@@ -0,0 +1,46 @@
+import * as R from "https://esm.sh/ramda";
+import { ParseResult } from "./types.ts";
+import { parseRow } from "./parseRow.ts";
+
+export type SchemaDefinition = {
+  constructors: ParseResult[];
+  methods: ParseResult[];
+  layerNumber: number;
+};
+
+export function getSchemaDefinition(schemaContent: string): SchemaDefinition {
+  const fileRows = schemaContent.split("\n").filter(Boolean);
+  const layerNumber = getLayerNumber(fileRows);
+
+  const { constructors, methods } = getConstructsAndMethodsFromRows(fileRows);
+
+  return {
+    constructors: constructors.map((row) => parseRow(row)),
+    methods: methods.map((row) => parseRow(row)),
+    layerNumber,
+  };
+}
+
+function getLayerNumber(rows: string[]) {
+  return +rows.find(r=>r.match(/LAYER \d+/))?.match(/\d+/)[0];
+}
+
+function getConstructsAndMethodsFromRows(rows: string[]) {
+  const fileRowsWithoutComments = rows
+    .filter((row) => !isComment(row))
+    .map((row) => row.trim());
+
+  const [constructsRows, methodRows] = R.splitAt(
+    fileRowsWithoutComments.indexOf("---functions---"),
+    fileRowsWithoutComments,
+  );
+  methodRows.shift();
+  return {
+    constructors: constructsRows,
+    methods: methodRows,
+  };
+}
+
+function isComment(row: string) {
+  return row.startsWith("//");
+}

type-generation/tl-parser/types.ts

@@ -0,0 +1,26 @@
+export type GenericParams = {
+  genericDescriptor?: { key: string; type: string }[];
+  argsOrder?: string[];
+};
+
+export type ParsedParam = {
+  name: string;
+  type: string;
+  index: number;
+  isOptional: boolean;
+};
+
+export type ParseResult = {
+  name: string;
+  id: number;
+  generic?: GenericParams;
+  params: ParsedParam[];
+  type: string;
+  subclassOfId: number;
+};
+
+export type ParsedSchema = {
+  constructors: ParseResult[];
+  methods: ParseResult[];
+  layerNumber: number;
+};

type-generation/tl-parser/utils.ts

@@ -0,0 +1,43 @@
+export function signedHexToInt(hex: string) {
+  if (hex.length % 2 != 0) {
+    hex = "0" + hex;
+  }
+  let num = parseInt(hex, 16);
+  const maxVal = Math.pow(2, hex.length / 2 * 8);
+  if (num > maxVal / 2 - 1) {
+    num = num - maxVal;
+  }
+  return num;
+}
+
+// Taken from https://stackoverflow.com/questions/18638900/javascript-crc32/18639999#18639999
+function makeCRCTable() {
+  let c;
+  const crcTable = [];
+  for (let n = 0; n < 256; n++) {
+    c = n;
+    for (let k = 0; k < 8; k++) {
+      c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1;
+    }
+    crcTable[n] = c;
+  }
+  return crcTable;
+}
+
+let crcTable: number[] | undefined = undefined;
+
+export function crc32(buf: Uint8Array | string) {
+  if (!crcTable) {
+    crcTable = makeCRCTable();
+  }
+  if (!(buf instanceof Uint8Array)) {
+    buf = new TextEncoder().encode(buf);
+  }
+  let crc = -1;
+
+  for (let index = 0; index < buf.length; index++) {
+    const byte = buf[index];
+    crc = crcTable[(crc ^ byte) & 0xff] ^ (crc >>> 8);
+  }
+  return (crc ^ -1) >>> 0;
+}