diff --git a/deployment/generate-tm.sh b/deployment/generate-tm.sh index 3527253d..f3abd3c4 100755 --- a/deployment/generate-tm.sh +++ b/deployment/generate-tm.sh @@ -3,7 +3,6 @@ ROOT="$1" PUBLIC="$2" PROGRAMS=$ROOT/tm-git/src BUILDER=$PROGRAMS -CHECK_LINKS=$ROOT/tm-git/integration-tests NEW_POS=$PROGRAMS/output # PUBLISHED directories are used to allow to publish the previous version if # we have been unable to fetch it. @@ -43,7 +42,7 @@ mkdir -p $PUBLISHED_PO mkdir -p $PUBLISHED_TMX # Check project links -cd $CHECK_LINKS +cd $PROGRAMS python run-check-links.py # Build new translation files @@ -55,7 +54,7 @@ rm -f -r $NEW_POS python builder.py -d python builder.py --softcatala cp builder-error.log applications-error.log -cat $CHECK_LINKS/run-check-links-error.log >> applications-error.log +cat $PROGRAMS/run-check-links-error.log >> applications-error.log copy_successfully_downloaded_files "*.po" 200 $PUBLISHED_PO diff --git a/src/run-check-links.py b/src/run-check-links.py new file mode 100755 index 00000000..a646f8d5 --- /dev/null +++ b/src/run-check-links.py @@ -0,0 +1,118 @@ +# -*- coding: utf-8 -*- +# +# Copyright (c) 2022 Jordi Mas i Hernandez +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +import sys +import os +import logging +from urllib.request import Request, urlopen +from urllib.error import HTTPError + +sys.path.append("../src/") + +from builder.jsonbackend import JsonBackend + +TIMEOUT = 10 +HTTP_STATUS_CODE_OK = 200 +HTTP_STATUS_CODE_NOT_FOUND = 404 + + +def init_logging(del_logs): + logfile = "run-check-links.log" + logfile_error = "run-check-links-error.log" + + if del_logs and os.path.isfile(logfile): + os.remove(logfile) + + if del_logs and os.path.isfile(logfile_error): + os.remove(logfile_error) + + formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s") + + LOGLEVEL = os.environ.get("LOGLEVEL", "INFO").upper() + LOGSTDOUT = os.environ.get("LOGSTDOUT", "0") + + if LOGSTDOUT == "0": + console = logging.StreamHandler() # By default uses stderr + else: + console = logging.StreamHandler(stream=sys.stdout) + + logging.basicConfig(filename=logfile, level=logging.DEBUG) + logger = logging.getLogger("") + console.setLevel(LOGLEVEL) + + if LOGLEVEL != "INFO": + console.setFormatter(formatter) + + logger.addHandler(console) + + fh = logging.FileHandler(logfile_error) + fh.setLevel(logging.ERROR) + fh.setFormatter(formatter) + logger.addHandler(fh) + + +def check_project_link(project_web): + if project_web is None or len(project_web) == 0: + return + + code = HTTP_STATUS_CODE_OK + try: + req = Request( + project_web, + headers={"User-Agent": "Mozilla/5.0 (X11; Linux x86_64;) Gecko Firefox"}, + ) + response = urlopen(req) + response.close() + + except HTTPError as e: + code = e.code + + except Exception as e: + logging.error(f"Project link {project_web} returns exception '{e}'") + return False + + if code != HTTP_STATUS_CODE_OK: + logging.error(f"Project link {project_web} returns {code}") + return False + else: + return True + + +def check_project_links(): + json = JsonBackend("../cfg/projects/") + json.load() + + checked = 0 + errors = 0 + for project_dto in json.projects: + if not project_dto.downloadable: + continue + + checked += 1 + if not check_project_link(project_dto.projectweb): + errors += 1 + + print(f"Links checked {checked}, with errors {errors}") + + +if __name__ == "__main__": + print("Check all projects links to external project webs") + + init_logging(True) + check_project_links()