From 4620200f155382af15848120259f7b1aae9f15f4 Mon Sep 17 00:00:00 2001 From: Dimitri Date: Sat, 30 Apr 2022 00:22:10 +0200 Subject: [PATCH] Matrix capability integration To support Matrix as an outlet for ghi we: -edited ghi/configuration.py to read, check and use the (optional) Matrix-settings from the .yml -created ghi/ghimatrix.py to facilitate creating credential-files, logging onto a Matrix server and sending messages to one or more rooms on it by using the matrix-nio module -changed some naming and matrix-specific things in most of the files -changed the README.md en .ghy.yml.example to include the new Matrix-support -created ghi/util.py to facilitate in a workaround for a Matrix-issue (https://github.com/matrix-org/matrix-appservice-irc/issues/1562) -added relevant exception-handlers No changes to the config-file are necessary when using IRC. It works quite well with server.py, haven't tested it with AWS though, but should work as well. Closes #27 Co-authored-by: W. J. van der Laan --- .ghi.yml.example | 37 +++++--- README.md | 124 ++++++++++++++++++--------- ghi/configuration.py | 170 +++++++++++++++++++++++++++++++++++-- ghi/events/pull_request.py | 20 ++++- ghi/events/push.py | 65 +++++++++++--- ghi/ghimatrix.py | 123 +++++++++++++++++++++++++++ ghi/github.py | 6 +- ghi/index.py | 65 +++++++++----- ghi/util.py | 10 +++ requirements-server.txt | 1 + 10 files changed, 525 insertions(+), 96 deletions(-) create mode 100644 ghi/ghimatrix.py create mode 100644 ghi/util.py diff --git a/.ghi.yml.example b/.ghi.yml.example index 9d12557..2ca23f1 100644 --- a/.ghi.yml.example +++ b/.ghi.yml.example @@ -1,23 +1,32 @@ version: 1 pools: - - name: pool-name + - name: "pool-name" github: repos: - - name: owner/repo - secret: abc123 + - name: "owner/repo" + secret: "abc123" irc: - host: chat.freenode.net - nick: my-irc-bot - password: myBotPassword123! + host: "chat.freenode.net" + nick: "my-irc-bot" + password: "myBotPassword123!" channels: - - channel1 + - "channel1" mastodon: - instance: https://mstdn.social - user: happy@place.net - password: myBotPassword123! - secretspath: /home/thatsme/my/secrets/ - appname: my-mastodon-bot + instance: "https://mstdn.social" + user: "happy@place.net" + password: "myBotPassword123!" + secretspath: "/home/thatsme/my/mastodonsecrets/" + appname: "my-mastodon-bot" merges_only: True + matrix: + homeserver: "https://a.matrix.srv" + user: "@ghibot:matrix.srv" + password: "anotherGreatPassword456!" + secretspath: "/home/thatsme/my/matrixsecrets/" + device_id: "Ghi-Matrix-Bot" + rooms: + - "#room:matrix.srv" outlets: - - irc - - mastodon \ No newline at end of file + - "irc" + - "mastodon" + - "matrix" \ No newline at end of file diff --git a/README.md b/README.md index dbc55c7..ca9d7fb 100644 --- a/README.md +++ b/README.md @@ -2,12 +2,12 @@ **G**it**H**ub **I**RC Notification Service -Ghi (pronounced 'ghee') is a relay between GitHub and IRC and/or Mastodon. It was created to take the place of the [now depreciated](https://developer.github.com/changes/2018-04-25-github-services-deprecation/) [GitHub IRC Service](https://github.com/github/github-services/blob/master/lib/services/irc.rb). Ghi receives events from GitHub for a specified repository via a webhook. Then it parses the event and sends the relevant information to your configured IRC channels and/or Mastodon timeline. Ghi was written to be very configuration driven. Therefore, Ghi is set up with a `.ghi.yml` file and can listen for multiple repositories and send to multiple IRC channels. Most of the features in the original GitHub Service are supported in Ghi as well. +Ghi (pronounced 'ghee') is a relay between GitHub and IRC and/or Mastodon. It was created to take the place of the [now depreciated](https://developer.github.com/changes/2018-04-25-github-services-deprecation/) [GitHub IRC Service](https://github.com/github/github-services/blob/master/lib/services/irc.rb). Ghi receives events from GitHub for a specified repository via a webhook. Then it parses the event and sends the relevant information to your configured IRC channels and/or Mastodon timeline and/or Matrix rooms. Ghi was written to be very configuration driven. Therefore, Ghi is set up with a `.ghi.yml` file and can listen for multiple repositories and send to multiple IRC channels. Most of the features in the original GitHub Service are supported in Ghi as well. # Getting Started -Ghi was designed and written to be ran in [AWS Lambda](https://aws.amazon.com/lambda/) with [API Gateway](https://aws.amazon.com/api-gateway/). However, I've also created a very simple HTTP server implementation so Ghi can be ran on any server if desired. Ghi is configured entirely with the `.ghi.yml` file. In this file you will set all necessary information including repositories, IRC nick, IRC host, channels, Mastodon instance, Mastodon user, etc. +Ghi was designed and written to be ran in [AWS Lambda](https://aws.amazon.com/lambda/) with [API Gateway](https://aws.amazon.com/api-gateway/). However, I've also created a very simple HTTP server implementation so Ghi can be ran on any server if desired. Ghi is configured entirely with the `.ghi.yml` file. In this file you will set all necessary information including repositories, IRC nick, IRC host, channels, Mastodon instance, Mastodon user, Matrix homeserver, Matrix rooms, etc. ## Deployment @@ -38,6 +38,14 @@ Ghi supports pushing messages to Mastodon. Since Ghi, as the name implies, is ma $ pip3 install mastodon-py ``` +### Matrix + +Ghi also supports pushing messages to Matrix. Again, since Ghi, as the name implies, is mainly focused on IRC the module requirement for Mastodon is optional. If you want to use Matrix as one of the outlets the matrix-nio module is required: + +``` +$ pip3 install matrix-nio +``` + ## Setting Configuration ### .ghi.yml @@ -56,7 +64,7 @@ To explain, if I have a `~/.ghi.yml` file and a `./.ghi.yml` file in my current #### Contents -The Ghi file is where you specify things like repositories, branches, channels, IRC details, etc. Ghi uses something called a "Pool" to determine which events do what. A Pool can have 1 or more repositories and 1 or more channels. You can also list multiple pools in a single Ghi instance. So you could have both `gkrizek/repo1` and `gkrizek/repo2` sending messages to `#my-cool-channel` while also having `gkrizek/repo3` sending messages to `#other-cool-channel` and `#last-cool-channel` (and of course many more variations of that). +The Ghi file is where you specify things like repositories, branches, channels, and IRC/Mastodon/Matrix details. Ghi uses something called a "Pool" to determine which events do what. A Pool can have 1 or more repositories and 1 or more channels. You can also list multiple pools in a single Ghi instance. So you could have both `gkrizek/repo1` and `gkrizek/repo2` sending messages to `#my-cool-channel` while also having `gkrizek/repo3` sending messages to `#other-cool-channel` and `#last-cool-channel` (and of course many more variations of that). The top two required parameters of the Ghi file are `version` and `pools`. Currently there is only a version `1` of the Ghi file, but `pools` will be a list of Pool configurations. Each Pool is required to define some GitHub information like repository names and validation secrets. They will also need to specify IRC data like nick, host, password, and channels. @@ -67,25 +75,34 @@ There are a lot more options that you can set to further configure Ghi. [See the ```yaml version: 1 pools: - - name: my-pool + - name: "my-pool" github: repos: - - name: gkrizek/repo1 - secret: 3ccb8d36bd4c67dd1dffcff9ca2c40 + - name: "gkrizek/repo1" + secret: "3ccb8d36bd4c67dd1dffcff9ca2c40" irc: - host: chat.freenode.net - nick: my-irc-bot + host: "chat.freenode.net" + nick: "my-irc-bot" channels: - - my-cool-channel + - "my-cool-channel" mastodon: - instance: https://mstdn.social - user: happy@place.net - password: myBotPassword123! - secretspath: /home/thatsme/my/secrets/ - appname: my-mastodon-bot + instance: "https://mstdn.social" + user: "happy@place.net" + password: "myBotPassword123!" + secretspath: "/home/thatsme/my/mastodonsecrets/" + appname: "my-mastodon-bot" + matrix: + homeserver: "https://a.matrix.srv" + user: "@ghibot:matrix.srv" + password: "anotherGreatPassword456!" + secretspath: "/home/thatsme/my/matrixsecrets/" + device_id: "Ghi-Matrix-Bot" + rooms: + - "#room:matrix.srv" outlets: - - irc - - mastodon + - "irc" + - "mastodon" + - "matrix" ``` _More Ghi file examples in [`examples/.ghi.yml.md`](examples/.ghi.yml.md)._ @@ -164,6 +181,11 @@ Ghi is configurable and supports lots of combinations of repositories, channels, | mastodon:secretspath | None | No | Path to Client and User Credential-Files (.secret) | | mastodon:appname | None | No | Name of the App for registration at the Instance | | mastodon:merges_only | True | No | Only toot merges to the Instance | +| matrix:homeserver | None | No | Hostname for Matrix Server | +| matrix:user | None | No | Matrix User | +| matrix:password | None | No | Matrix Password | +| matrix:secretspath | None | No | Path to Matrix` User Credential-Files (.json) | +| matrix:device_id | "Ghi-Matrix-Bot" | No | Name of the device(/app) being shown at the userinfo | **Pool Configuration Object** @@ -185,8 +207,14 @@ Ghi is configurable and supports lots of combinations of repositories, channels, | mastodon:secretspath | None | Yes | Path to Client and User Credential-Files (.secret) | | mastodon:appname | None | Yes | Name of the App for registration at the Instance | | mastodon:merges_only | True | No | Only toot merges to the Instance | +| matrix:homeserver | None | Yes | Hostname for Matrix Server | +| matrix:user | None | Yes | Matrix User | +| matrix:password | None | Yes | Matrix Password | +| matrix:secretspath | None | Yes | Path to Matrix' User Credential-Files (.json) | +| matrix:device_id | "Ghi-Matrix-Bot" | Yes | Name of the device(/app) being shown at the userinfo | +| matrix:rooms | None | Yes | List of rooms IDs or room aliases | -~ For irc:* and mastodon:* : if they're one of the configured outlets. +~ For all irc, mastodon, and matrix settings: if they're one of the configured outlets. **Repository Configuration Object** @@ -208,51 +236,67 @@ global: # optional shorten_url: true # optional, defaults to false verify: true # optional, defaults to true irc: # optional - host: chat.freenode.net # optional, but must be set in pool if not here and needed (i.e. IRC is one of the outlets) + host: "chat.freenode.net" # optional, but must be set in pool if not here port: 6697 # optional, default is 6697 for ssl and 6667 for non-ssl ssl: true # optional, default is true - nick: my-irc-bot # optional, but must be set in pool if not here and needed (i.e. IRC is one of the outlets) - password: abc123 # optional, but must be set in pool if not here and needed for the nick that's used + nick: "my-irc-bot" # optional, but must be set in pool if not here + password: "abc123" # optional, but must be set in pool if not here and if needed for the nick that's used mastodon: # optional - instance: https://mstdn.social # optional, but must be set in pool if not here and needed (i.e. Mastodon is one of the outlets) - user: happy@place.net # optional, but must be set in pool if not here and needed (i.e. Mastodon is one of the outlets) - password: myBotPassword123! # optional, but must be set in pool if not here and needed (i.e. Mastodon is one of the outlets) - secretspath: /home/thatsme/my/secrets/ # optional, but must be set in pool if not here and needed (i.e. Mastodon is one of the outlets) - appname: my-mastodon-bot # optional, but must be set in pool if not here and needed (i.e. Mastodon is one of the outlets) + instance: "https://mstdn.social" # optional, but must be set in pool if not here + user: "happy@place.net" # optional, but must be set in pool if not here + password: "myBotPassword123!" # optional, but must be set in pool if not here + secretspath: "/home/thatsme/my/mastodonsecrets/" # optional, but must be set in pool if not here + appname: "my-mastodon-bot" # optional, but must be set in pool if not here merges_only: true # optional, default is true + matrix: # optional + homeserver: "https://a.matrix.srv" # optional, but must be set in pool if not here + user: "@ghibot:matrix.srv" # optional, but must be set in pool if not here + password: "anotherGreatPassword456!" # optional, but must be set in pool if not here + secretspath: "/home/thatsme/my/matrixsecrets/" # optional, but must be set in pool if not here + device_id: "Ghi-Matrix-Bot" # optional, default is "Ghi-Matrix-Bot" outlets: # optional, default is irc - - irc - - mastodon + - "irc" + - "mastodon" + - "matrix" pools: # required - - name: my-pool # required + - name: "my-pool" # required github: # required repos: # required - - name: gkrizek/repo1 # at least 1 repo is required - secret: 3ccb8d36bd4c67dd1dffcff9ca2c40 # optional, but if it's needed it must be set here or with environment variable + - name: "gkrizek/repo1" # at least 1 repo is required + secret: "3ccb8d36bd4c67dd1dffcff9ca2c40" # optional, but if it's needed it must be set here or with environment variable branches: # optional, default is 'all' - - master - - staging + - "master" + - "staging" verify: true # optional, default is true shorten_url: true # optional, defaults to false irc: # required if IRC is one of the outlets - host: chat.freenode.net # required + host: "chat.freenode.net" # required port: 6697 # optional, default is 6697 for ssl and 6667 for non-ssl ssl: true # optional, default is true - nick: my-irc-bot # required - password: abc123 # optional, but if it's required by the nick it must be set here or with environment variable + nick: "my-irc-bot" # required + password: "abc123" # optional, but if it's required by the nick it must be set here or with environment variable channels: # required - - my-cool-channel # at least 1 channel is required + - "my-cool-channel" # at least 1 channel is required mastodon: # required if Mastodon is one of the outlets instance: https://mstdn.social # required user: happy@place.net # required password: myBotPassword123! # required - secretspath: /home/thatsme/my/secrets/ # required - appname: my-mastodon-bot # required + secretspath: "/home/thatsme/my/mastodonsecrets/" # required + appname: "my-mastodon-bot" # required merges_only: true # optional, default is true + matrix: # required if Matrix is one of the outlets + homeserver: "https://a.matrix.srv" # required + user: "@ghibot:matrix.srv" # required + password: "anotherGreatPassword456!" # required + secretspath: "/home/thatsme/my/matrixsecrets/" # required + device_id: "Ghi-Matrix-Bot" # required, default is "Ghi-Matrix-Bot" + rooms: # required + - "#room:matrix.srv" # at least 1 room is required outlets: # optional, default is irc - - irc - - mastodon + - "irc" + - "mastodon" + - "matrix" ``` If you define a parameter in the Global section and in your pool, the value in the pool will be used. diff --git a/ghi/configuration.py b/ghi/configuration.py index 901749d..72cce5a 100644 --- a/ghi/configuration.py +++ b/ghi/configuration.py @@ -3,13 +3,15 @@ import os import yaml -SUPPORTED_OUTLETS = ["irc", "mastodon"] +SUPPORTED_OUTLETS = ["irc", "mastodon", "matrix"] +MATRIX_DEVICE_ID = "Ghi-Matrix-Bot" class Pool(object): def __init__(self, name, outlets, repos, shorten, ircHost, ircPort, ircSsl, ircNick, ircPassword, ircChannels,\ - mastInstance, mastUser, mastPassword, mastSecPath, mastAppName, mastMergeFilter): + mastInstance, mastUser, mastPassword, mastSecPath, mastAppName, mastMergeFilter,\ + matrixUser, matrixPassword, matrixServer, matrixRooms, matrixSecPath, matrixDevId): self.name = name self.outlets = outlets self.repos = repos @@ -26,10 +28,16 @@ def __init__(self, name, outlets, repos, shorten, ircHost, ircPort, ircSsl, ircN self.mastSecPath = mastSecPath self.mastAppName = mastAppName self.mastMergeFilter = mastMergeFilter + self.matrixUser = matrixUser + self.matrixPassword = matrixPassword + self.matrixServer = matrixServer + self.matrixRooms = matrixRooms + self.matrixSecPath = matrixSecPath + self.matrixDevId = matrixDevId def containsRepo(self, repo): - for configRepo in self.repos: + for configRepo in self.repos: if repo == configRepo["name"]: return True return False @@ -44,7 +52,8 @@ class GlobalConfig(object): def __init__(self, ircHost, ircPort, ircSsl, ircNick, ircPassword, mastInstance, mastUser,\ - mastPassword, mastSecPath, mastAppName, mastMergeFilter, shorten, verify, outlets): + mastPassword, mastSecPath, mastAppName, mastMergeFilter, shorten, verify, outlets,\ + matrixUser, matrixPassword, matrixServer, matrixSecPath, matrixDevId): self.ircHost = ircHost self.ircPort = ircPort self.ircSsl = ircSsl @@ -59,6 +68,11 @@ def __init__(self, ircHost, ircPort, ircSsl, ircNick, ircPassword, mastInstance, self.shorten = shorten self.verify = verify self.outlets = outlets + self.matrixUser = matrixUser + self.matrixPassword = matrixPassword + self.matrixServer = matrixServer + self.matrixSecPath = matrixSecPath + self.matrixDevId = matrixDevId def getConfiguration(): @@ -242,6 +256,48 @@ def getConfiguration(): globalMastAppName = None globalMastMergeFilter = None + if "matrix" in globalConfig: + if "user" in globalConfig["matrix"]: + globalMatrixUser = globalConfig["matrix"]["user"] + if type(globalMatrixUser) is not str: + raise TypeError("'user' is not a string") + else: + globalMatrixUser = None + + if "password" in globalConfig["matrix"]: + globalMatrixPassword = globalConfig["matrix"]["password"] + if type(globalMatrixPassword) is not str: + raise TypeError("'password' is not a string") + else: + globalMatrixPassword = None + + if "homeserver" in globalConfig["matrix"]: + globalMatrixServer = globalConfig["matrix"]["homeserver"] + if type(globalMatrixServer) is not str: + raise TypeError("'homeserver' is not a string") + else: + globalMatrixServer = None + + if "secretspath" in globalConfig["matrix"]: + globalMatrixSecPath = globalConfig["matrix"]["secretspath"] + if type(globalMatrixSecPath) is not str: + raise TypeError("'secretspath' is not a string") + else: + globalMatrixSecPath = None + + if "device_id" in globalConfig["matrix"]: + globalMatrixDevId = globalConfig["matrix"]["device_id"] + if type(globalMatrixDevId) is not str: + raise TypeError("'device_id' is not a string") + else: + globalMatrixDevId = MATRIX_DEVICE_ID + else: + globalMatrixUser = None + globalMatrixPassword = None + globalMatrixServer = None + globalMatrixSecPath = None + globalMatrixDevId = None + if "github" in globalConfig: if "shorten_url" in globalConfig["github"]: globalShorten = globalConfig["github"]["shorten_url"] @@ -292,6 +348,11 @@ def getConfiguration(): mastSecPath = globalMastSecPath, mastAppName = globalMastAppName, mastMergeFilter = globalMastMergeFilter, + matrixUser = globalMatrixUser, + matrixPassword = globalMatrixPassword, + matrixServer = globalMatrixServer, + matrixSecPath = globalMatrixSecPath, + matrixDevId = globalMatrixDevId, shorten = globalShorten, verify = globalVerify, outlets = globalGeneratedOutlets @@ -575,6 +636,91 @@ def getConfiguration(): else: mastMergeFilter = True + if "matrix" in generatedOutlets and "matrix" in pool: + if "user" in pool["matrix"]: + matrixUser = pool["matrix"]["user"] + elif globalSettings.matrixUser: + matrixUser = globalSettings.matrixUser + else: + raise KeyError("user") + if type(matrixUser) is not str: + raise TypeError("'user' is not a string") + + if "password" in pool["matrix"]: + matrixPassword = pool["matrix"]["password"] + elif globalSettings.matrixPassword: + matrixPassword = globalSettings.matrixPassword + else: + raise KeyError("password") + if type(matrixPassword) is not str: + raise TypeError("'password' is not a string") + + if "homeserver" in pool["matrix"]: + matrixServer = pool["matrix"]["homeserver"] + elif globalSettings.matrixServer: + matrixServer = globalSettings.matrixServer + else: + raise KeyError("homeserver") + if type(matrixServer) is not str: + raise TypeError("'homeserver' is not a string") + + if "secretspath" in pool["matrix"]: + matrixSecPath = pool["matrix"]["secretspath"] + elif globalSettings.matrixSecPath: + matrixSecPath = globalSettings.matrixSecPath + else: + raise KeyError("secretspath") + if type(matrixSecPath) is not str: + raise TypeError("'secretspath' is not a string") + + if "device_id" in pool["matrix"]: + matrixDevId = pool["matrix"]["device_id"] + elif globalSettings.matrixDevId: + matrixDevId = globalSettings.matrixDevId + else: + matrixDevId = MATRIX_DEVICE_ID + if type(matrixDevId) is not str: + raise TypeError("'device_id' is not a string") + + if "rooms" in pool["matrix"]: + matrixRooms = pool["matrix"]["rooms"] + else: + raise KeyError("rooms") + if type(matrixRooms) is not list: + raise TypeError("'rooms' is not a list") + if len(matrixRooms) < 1: + raise TypeError("'rooms' must contain at least 1 item") + + generatedMatrixRooms = list() + for room in matrixRooms: + generatedMatrixRooms.append(room)#"+room) + + elif "matrix" in generatedOutlets and "matrix" in globalConfig: + if globalSettings.matrixUser: + matrixUser = globalSettings.matrixUser + else: + raise KeyError("user") + + if globalSettings.matrixPassword: + matrixPassword = globalSettings.matrixPassword + else: + raise KeyError("password") + + if globalSettings.matrixServer: + matrixServer = globalSettings.matrixServer + else: + raise KeyError("homeserver") + + if globalSettings.matrixSecPath: + matrixSecPath = globalSettings.matrixSecPath + else: + raise KeyError("secretspath") + + if globalSettings.matrixDevId: + matrixDevId = globalSettings.matrixDevId + else: + matrixDevId = MATRIX_DEVICE_ID + except (KeyError, TypeError) as e: errorMessage = "Missing or invalid parameter in configuration file: %s" % e logging.error(errorMessage) @@ -602,6 +748,14 @@ def getConfiguration(): mastAppName = None mastMergeFilter = None + if "matrix" not in generatedOutlets: + matrixUser = None + matrixPassword = None + matrixServer = None + generatedMatrixRooms = None + matrixSecPath = None + matrixDevId = None + pools.append( Pool( name=name, @@ -619,7 +773,13 @@ def getConfiguration(): mastPassword=mastPassword, mastSecPath=mastSecPath, mastAppName=mastAppName, - mastMergeFilter=mastMergeFilter + mastMergeFilter=mastMergeFilter, + matrixUser=matrixUser, + matrixPassword=matrixPassword, + matrixServer=matrixServer, + matrixRooms=generatedMatrixRooms, + matrixSecPath=matrixSecPath, + matrixDevId=matrixDevId ) ) diff --git a/ghi/events/pull_request.py b/ghi/events/pull_request.py index 1047e1b..c348ce1 100644 --- a/ghi/events/pull_request.py +++ b/ghi/events/pull_request.py @@ -5,6 +5,7 @@ sys.path.append(os.path.dirname(os.path.realpath(__file__)) + "/../") import github from irc import Colors +from util import matrix_html def PullRequest(payload, shorten): @@ -53,10 +54,25 @@ def PullRequest(payload, shorten): url = url ) + matrixMessage = ( + '[{repo}] {user} {action} pull request #{number}: {title} ' + '({baseBranch}...{headBranch}) {url}' + ).format( + repo = matrix_html(payload["pull_request"]["base"]["repo"]["name"]), + user = matrix_html(payload["sender"]["login"]), + action = matrix_html(action), + number = payload["number"], + title = matrix_html(payload["pull_request"]["title"]), + baseBranch = matrix_html(payload["pull_request"]["base"]["ref"]), + headBranch = matrix_html(payload["pull_request"]["head"]["ref"]), + url = url + ) + return { "statusCode": 200, "ircMessages": [ircMessage], - "mastMessages": [mastMessage] + "mastMessages": [mastMessage], + "matrixMessages": [matrixMessage] } else: @@ -68,4 +84,4 @@ def PullRequest(payload, shorten): "success": True, "message": message }) - } \ No newline at end of file + } diff --git a/ghi/events/push.py b/ghi/events/push.py index bc4952d..42c564e 100644 --- a/ghi/events/push.py +++ b/ghi/events/push.py @@ -5,6 +5,7 @@ sys.path.append(os.path.dirname(os.path.realpath(__file__)) + "../") import github from irc import Colors +from util import matrix_html def Push(payload, poolRepos, shorten): @@ -54,22 +55,35 @@ def Push(payload, poolRepos, shorten): compareUrl = url ) + matrixMessage = ( + '[{repo}] {user} {action} ' + 'tag {tag}: {compareUrl}' + ).format( + repo = matrix_html(payload["repository"]["name"]), + user = matrix_html(payload["pusher"]["name"]), + action = matrix_html(action), + tag = matrix_html(ref.split("/", maxsplit=2)[2]), + compareUrl = url + ) + return { "statusCode": 200, "ircMessages": [ircMessage], - "mastMessages": [mastMessage] + "mastMessages": [mastMessage], + "matrixMessages": [matrixMessage] } else: # Commits were pushed - ircMessages = [] - mastMessages = [] - commits = payload["commits"] - repo = payload["repository"]["name"] - fullName = payload["repository"]["full_name"] - user = payload["pusher"]["name"] - length = len(commits) - branch = ref.split("/", maxsplit=2)[2] + ircMessages = [] + mastMessages = [] + matrixMessages = [] + commits = payload["commits"] + repo = payload["repository"]["name"] + fullName = payload["repository"]["full_name"] + user = payload["pusher"]["name"] + length = len(commits) + branch = ref.split("/", maxsplit=2)[2] # Check if the pool has allowed branches set. # If they do, make sure that this branch is included @@ -143,13 +157,28 @@ def Push(payload, poolRepos, shorten): ) ) + matrixMessages.append( + '[{repo}] {user} {action} ' + '{length} commit{plural} to {branch}: ' + '{compareUrl}'.format( + repo = matrix_html(repo), + user = matrix_html(user), + action = matrix_html(action), + length = length, + branch = matrix_html(branch), + compareUrl = url, + plural = matrix_html(plural) + ) + ) + # First 3 individual commits num = 0 for commit in commits: if num > 2: break - # If commit message is longer than 75 characters, truncate. - commitMessage = commit["message"] + # We're only interested in the first line of the commit message, the title. + # If it is longer than 75 characters, truncate. + commitMessage = commit["message"].split('\n', 1)[0] author = commit["author"]["name"] if len(commitMessage) > 75: commitMessage = commitMessage[0:74] + "..." @@ -179,10 +208,22 @@ def Push(payload, poolRepos, shorten): ) ) + matrixMessages.append( + '{repo}/{branch} ' + '{shortCommit} {user}: {message}'.format( + repo = repo, + branch = branch, + shortCommit = commit["id"][0:7], + user = author, + message = commitMessage + ) + ) + num += 1 return { "statusCode": 200, "ircMessages": ircMessages, - "mastMessages": mastMessages + "mastMessages": mastMessages, + "matrixMessages": matrixMessages } diff --git a/ghi/ghimatrix.py b/ghi/ghimatrix.py new file mode 100644 index 0000000..aab9ef0 --- /dev/null +++ b/ghi/ghimatrix.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 + +import asyncio +import json +import os +import sys +import logging + +try: + from nio import AsyncClient, LoginResponse + from nio.responses import RoomResolveAliasError +except ImportError: + pass + + +CRED_FILE = "ghi_matrix_credentials.json" + + +def createCreds(resp: LoginResponse, homeserver, matrixSecFile) -> None: + """Writes the required login details to disk so we can log in later without + using a password. + + Arguments: + resp {LoginResponse} -- the successful client login response. + homeserver -- URL of homeserver, e.g. "https://matrix.example.org" + """ + # open the config file in write-mode + with open(matrixSecFile, "w") as f: + # write the login details to disk + json.dump( + { + "homeserver": homeserver, + "user_id": resp.user_id, + "device_id": resp.device_id, + "access_token": resp.access_token, + }, + f, + ) + + +def sendMessages(*args, **kwargs) -> None: + return asyncio.run(_sendMessages(*args, **kwargs)) + + +async def _sendMessages(pool, messages) -> None: + homeserver = pool.matrixServer + user_id = pool.matrixUser + password = pool.matrixPassword + deviceName = pool.matrixDevId + credPath = pool.matrixSecPath + rooms = pool.matrixRooms + + matrixSecFile = os.path.join(credPath, CRED_FILE) + # If there are no previously-saved credentials, we'll use the password + if not os.path.exists(matrixSecFile): + logging.info("First time use. Did not find credential file. Using info from config to create one.") + + if not (homeserver.startswith("https://") or homeserver.startswith("http://")): + homeserver = "https://" + homeserver + + client = AsyncClient(homeserver, user_id) + + resp = await client.login(password, device_name=deviceName) + + # check that we logged in successfully + if isinstance(resp, LoginResponse): + createCreds(resp, homeserver, matrixSecFile) + else: + logging.info(f'homeserver = "{homeserver}"; user = "{user_id}"') + logging.info(f"Failed to log in: {resp}") + sys.exit(1) + + logging.info("Logged in using a password. Credentials were stored.") + + await client.close() + + with open(matrixSecFile, "r") as f: + config = json.load(f) + client = AsyncClient(config["homeserver"]) + + client.access_token = config["access_token"] + client.user_id = config["user_id"] + client.device_id = config["device_id"] + + for room_id in rooms: + await client.join(room_id) + if room_id[0] != "!": + response = await client.room_resolve_alias(room_id) + + if isinstance(response, RoomResolveAliasError): + logging.info(f"Error looking up alias for {room_id}: {response}") + return { + "statusCode": 500, + "body": json.dumps({ + "success": False, + "message": "An error happened while sending messages to Matrix" + }) + } + + room_id = response.room_id + + for message in messages: + await client.room_send( + room_id, + message_type="m.room.message", + content={"msgtype": "m.text", "body": "", "format": "org.matrix.custom.html", "formatted_body":message}, + ) + + await client.close() + + if len(messages) == 1: + resultMessage = "Matrix - Successfully sent 1 message." + else: + resultMessage = "Matrix - Successfully sent {} messages.".format(len(messages)) + + logging.info(resultMessage) + return { + "statusCode": 200, + "body": json.dumps({ + "success": True, + "message": resultMessage + }) + } diff --git a/ghi/github.py b/ghi/github.py index 7373c05..14bb5a9 100644 --- a/ghi/github.py +++ b/ghi/github.py @@ -82,7 +82,8 @@ def parsePayload(event, payload, repos, shorten): return { "statusCode": 200, "ircMessages": push["ircMessages"], - "mastMessages": push["mastMessages"] + "mastMessages": push["mastMessages"], + "matrixMessages": push["matrixMessages"] } @@ -95,7 +96,8 @@ def parsePayload(event, payload, repos, shorten): return { "statusCode": 200, "ircMessages": pullRequest["ircMessages"], - "mastMessages": pullRequest["mastMessages"] + "mastMessages": pullRequest["mastMessages"], + "matrixMessages": pullRequest["matrixMessages"] } diff --git a/ghi/index.py b/ghi/index.py index 0539ede..8f5a868 100644 --- a/ghi/index.py +++ b/ghi/index.py @@ -3,15 +3,31 @@ sys.path.append(os.path.dirname(os.path.realpath(__file__))) import json import logging + from configuration import getConfiguration from github import getPool, parsePayload -from irc import sendMessages from ghilogging import setup_server_logging +from irc import sendMessages as sendIrcMessages from ghimastodon import sendToots +from ghimatrix import sendMessages as sendMatrixMessages from validation import validatePayload from __init__ import __version__ +def composeResultMessage(outlets): + result = "Successfully notified " + + if len(outlets) == 1: + result += outlets[0] + elif len(outlets) == 2: + result += "both " + outlets[0] + " and " + outlets[1] + else: + ", and ".join(outlets) + result += "." + + return result + + def handler(event, context=None, sysd=None): # ensure it's a valid request if event and "body" in event and "headers" in event: @@ -96,24 +112,22 @@ def handler(event, context=None, sysd=None): if getMessages["statusCode"] != 200: return getMessages - ircCheck = False - mastCheck = False + outletChecks = list() failure = False - + if "irc" in pool["pool"].outlets: logging.debug("IRC Messages:") logging.debug(getMessages["ircMessages"]) # Send messages to the designated IRC channel(s) - sendToIrc = sendMessages(pool["pool"], getMessages["ircMessages"]) + sendToIrc = sendIrcMessages(pool["pool"], getMessages["ircMessages"]) if sendToIrc["statusCode"] != 200: failure = True ircResult = "Something went wrong while trying to notify IRC." else: ircResult = "Successfully notified IRC." - ircCheck = True + outletChecks.append("IRC") logging.info(ircResult) - githubPayload = json.loads(githubPayload) @@ -135,22 +149,31 @@ def handler(event, context=None, sysd=None): failure = True mastResult = "Something went wrong while trying to notify Mastodon." else: - mastResult = "Succesfully notified Mastodon." - mastCheck = True + mastResult = "Successfully notified Mastodon." + outletChecks.append("Mastodon") logging.info(mastResult) - - if ircCheck or not mastAppliedMergeFilter and not failure: - result = "Succesfully notified {both0}{IRC}{both1}{Mastodon}.".format( - both0 = "both " if ircCheck and mastCheck else "", - both1 = " and " if ircCheck and mastCheck else "", - IRC = "IRC" if ircCheck else "", - Mastodon = "Mastodon" if mastCheck else "" - ) + + if "matrix" in pool["pool"].outlets: + logging.debug("Matrix Messages:") + logging.debug(getMessages["matrixMessages"]) + + # Send messages to the designated Matrix' room(s) + sendToMatrix = sendMatrixMessages(pool["pool"], getMessages["matrixMessages"]) + if sendToMatrix["statusCode"] != 200: + failure = True + matrixResult = "Something went wrong while trying to notify Matrix." + else: + matrixResult = "Successfully notified Matrix." + outletChecks.append("Matrix") + logging.info(matrixResult) + + if "IRC" in outletChecks or not mastAppliedMergeFilter or "Matrix" in outletChecks and not failure: + result = composeResultMessage(outletChecks) if "mastodon" in pool["pool"].outlets and mastAppliedMergeFilter: mastResult = "Didn't toot because of the merge filter." logging.info(mastResult) - result = result[:-1] + ", but not Mastodon because of the merge filter." - elif "mastodon" in pool["pool"].outlets and mastAppliedMergeFilter: + result = result[:-1] + ", but not Mastodon because of the merge filter." + elif "mastodon" in pool["pool"].outlets and mastAppliedMergeFilter and not failure: mastResult = "Event received, but didn't toot because of the merge filter." logging.info(mastResult) result = "Event received, but didn't toot because of the merge filter." @@ -164,7 +187,7 @@ def handler(event, context=None, sysd=None): "message": result }) } - else: + else: return { "statusCode": 200, "body": json.dumps({ @@ -180,4 +203,4 @@ def handler(event, context=None, sysd=None): "success": False, "message": "bad event data" }) - } \ No newline at end of file + } diff --git a/ghi/util.py b/ghi/util.py new file mode 100644 index 0000000..6de9901 --- /dev/null +++ b/ghi/util.py @@ -0,0 +1,10 @@ +def matrix_html(msg): + ''' + Escape text for matrix HTML. + ''' + # This should be html.escape. however, the Matrix→IRC bridge currently eats + # everything that looks like HTML, even escaped. So go for a solution that + # works for our particular case: simply strip characters that the bridge + # has difficulty with (escaped and unescaped). + # See https://github.com/matrix-org/matrix-appservice-irc/issues/1562. + return msg.replace('<', '').replace('>', '').replace('<', '').replace('>', '') diff --git a/requirements-server.txt b/requirements-server.txt index 1a1a99d..b130ef3 100644 --- a/requirements-server.txt +++ b/requirements-server.txt @@ -2,3 +2,4 @@ PyYAML~=5.4 requests~=2.26 tornado~=6.1 mastodon-py~=1.5 +matrix-nio~=0.19