Managed to get the whole thing working with docker compose

.gitattributes

@@ -0,0 +1,2 @@
+users.json filter=lfs diff=lfs merge=lfs -text
+projects.json filter=lfs diff=lfs merge=lfs -text

api/Dockerfile

@@ -3,7 +3,7 @@ FROM python:3.8-slim-buster
 # https://smirnov-am.github.io/running-flask-in-production-with-docker/
 # https://abdul-the-coder.medium.com/how-to-deploy-a-containerised-python-web-application-with-docker-flask-and-uwsgi-8862a08bd5df
 
-RUN apt update && apt install git gcc nginx build-essential systemd systemd-sysv -y
+RUN apt update && apt install git gcc nginx build-essential systemd systemd-sysv curl procps lsof -y
 
 RUN pip install uwsgi
 
@@ -23,7 +23,6 @@ ENV PYHTONUNBUFFERED 1
 # set user to `user`
 USER user
 
-
 RUN mkdir /home/user/code
 WORKDIR /home/user/code
 ADD ./requirements.txt ./

api/Makefile

@@ -20,7 +20,7 @@ install-deps: venv $(CONFIG_FILE)
 
 ##run: run the api locally
 run: install-deps
-	GOOGLE_APPLICATION_CREDENTIALS=.secrets/service-account.json $(PYTHON) manage.py run
+	$(PYTHON) manage.py run
 
 lint: venv
 	$(PYTHON) -m flake8 . --show-source --statistics

api/app/main/utils/database/search_projects.py

@@ -11,7 +11,7 @@
 
 def get_search_projects(query: str, count: int = 20, page: int = 1):
     """
-    get_search_users [this method search for users in our datastore
+    get_search_projects [this method search for projects in our datastore
 
     @params : query, count, page
     @returns : - code : the status code of the request
@@ -22,13 +22,16 @@ def get_search_projects(query: str, count: int = 20, page: int = 1):
     offset = (page - 1) * count
 
     client = meilisearch.Client(MEILISEARCH_HOST, MEILISEARCH_MASTER_KEY)
+    client.index(storage.KIND_PROJECTS).update_pagination_settings({'maxTotalHits': 5000})
     index = client.get_index(storage.KIND_PROJECTS)
     ret = index.search(
         storage.KIND_PROJECTS, {"q": query, "limit": count, "offset": offset}
     )
     if not ret or len(ret) < 1:
         return {"code": 400, "reason": "nothing found"}
 
+    ret["nbHits"] = ret["estimatedTotalHits"]
+
     response = {
         "code": 200,
         "status": "success",
@@ -47,7 +50,7 @@ def build_project_filters(languages):
     for i in range(len_lang):
         langs += f'language = "{languages[i]}"'
         if i != len_lang - 1:
-            langs += " OR "
+            langs += " "
     filters = langs
     return filters
 
@@ -98,6 +101,8 @@ def post_search_projects(
     offset = (page - 1) * count
     filters = build_project_filters(languages)
     client = meilisearch.Client(MEILISEARCH_HOST, MEILISEARCH_MASTER_KEY)
+    client.index(storage.KIND_PROJECTS).update_pagination_settings({'maxTotalHits': 5000})
+
     index = client.get_index(storage.KIND_PROJECTS)
 
     # if sort_type is not specified or not supported
@@ -108,7 +113,7 @@ def post_search_projects(
     ]:
         query_object = {"q": query, "limit": count, "offset": offset}
         if filters != "":
-            query_object["filters"] = filters
+            query_object["q"] += " " + filters
         ret = index.search(
             storage.KIND_PROJECTS,
             query_object,
@@ -120,7 +125,7 @@ def post_search_projects(
     else:
         query_object = {"q": query, "limit": 1000}
         if filters != "":
-            query_object["filters"] = filters
+            query_object["q"] += " " + filters
         ret = index.search(storage.KIND_PROJECTS, query_object)
         if not ret or len(ret) < 1:
             return {"code": 400, "reason": "nothing found"}
@@ -129,6 +134,8 @@ def post_search_projects(
         ret["offset"] = offset
         ret["limit"] = count
 
+    ret["nbHits"] = ret["estimatedTotalHits"]
+
     response = {
         "code": 200,
         "status": "success",

api/app/main/utils/database/search_users.py

@@ -22,13 +22,16 @@ def get_search_users(query: str, count: int = 20, page: int = 1):
     offset = (page - 1) * count
 
     client = meilisearch.Client(MEILISEARCH_HOST, MEILISEARCH_MASTER_KEY)
+    client.index(storage.KIND_USERS).update_pagination_settings({'maxTotalHits': 5000})
     index = client.get_index(storage.KIND_USERS)
     ret = index.search(
         storage.KIND_USERS, {"q": query, "limit": count, "offset": offset}
     )
     if not ret or len(ret) < 1:
         return {"code": 400, "reason": "nothing found"}
 
+    ret["nbHits"] = ret["estimatedTotalHits"]
+
     response = {
         "code": 200,
         "status": "success",
@@ -75,6 +78,7 @@ def post_search_users(
 
     offset = (page - 1) * count
     client = meilisearch.Client(MEILISEARCH_HOST, MEILISEARCH_MASTER_KEY)
+    client.index(storage.KIND_USERS).update_pagination_settings({'maxTotalHits': 5000})
     index = client.get_index(storage.KIND_USERS)
 
     # if sort_type is not specified or not supported
@@ -101,6 +105,8 @@ def post_search_users(
         ret["offset"] = offset
         ret["limit"] = count
 
+    ret["nbHits"] = ret["estimatedTotalHits"]
+
     response = {
         "code": 200,
         "status": "success",

api/app/main/utils/database/storage.py

@@ -1,15 +1,2 @@
-from google.cloud import datastore
-
-# For the gcloud auth to work properly this env variable should be set
-# GOOGLE_APPLICATION_CREDENTIALS=.secrets/service-account.json
-
 KIND_USERS = "github_users"
 KIND_PROJECTS = "github_projects"
-__CLIENT = None
-
-
-def get_client():
-    global __CLIENT
-    if __CLIENT is None:
-        __CLIENT = datastore.Client()
-    return __CLIENT

api/app/main/utils/database/twitter/top_tweets.py

@@ -1,10 +1,3 @@
-import requests
-from requests_oauthlib import OAuth1
-from app.settings import API_KEY, API_SECRET_KEY
-from app.main.utils.helpers.commons import get_trace
-import json
-
-
 def top_tweets(cache: object, count: int):
     """
     This method will return top-tweets
@@ -14,28 +7,7 @@ def top_tweets(cache: object, count: int):
     return boolean telling if everything went well [top-tweets] as string
     """
 
-    if cache.get("top-tweets") is None:
-        try:
-            # we make another request
-            # to the twitter api
-            print(">> Hitting twitter api...")
-
-            search_twitter_host = "https://api.twitter.com/1.1/search/tweets.json"
-            tweets = requests.get(
-                "{}?q=%23caparledev%20-filter%3Aretweets&count={}".format(search_twitter_host, str(count)),
-                auth=OAuth1(API_KEY, API_SECRET_KEY)
-            ).content.decode()
-
-            # and we cache it as json string for 1h
-            cache.set("top-tweets", tweets, 3600)
-        except Exception:
-            # We just print the trace-back here
-            get_trace()
-            return (False, cache.get("top-tweets"))
-    else:
-        print("<< Getting from cache...")
-
-    return (True, cache.get("top-tweets"))
+    return ""
 
 
 def get_top_tweets(cache: object, count: int):
@@ -44,25 +16,4 @@ def get_top_tweets(cache: object, count: int):
     the appropriate status code for the request
 
     """
-
-    results = top_tweets(cache, count)
-
-    if results[0]:
-        payload = json.loads(results[1])
-        if "errors" in payload:
-            return {
-                "code": 500,
-                "status": "error",
-                "result": payload
-            }
-        else:
-            return {
-                "code": 200,
-                "status": "success",
-                "result": payload
-            }
-    else:
-        return {
-            "code": 500,
-            "status": "error"
-        }
+    return ""

api/requirements.txt

@@ -1,22 +1,38 @@
-# prod deps
+aniso8601==9.0.1
+annotated-types==0.7.0
+attrs==24.2.0
+camel-converter==3.1.2
+certifi==2024.7.4
+charset-normalizer==3.3.2
+click==7.1.2
+flake8==4.0.1
 Flask==1.1.4
-Flask-Cors==3.0.10
+Flask-Cors==4.0.1
+flask-restplus==0.13.0
 Flask-Script==2.0.6
-Flask-Restplus==0.13.0
-google-api-core==2.3.0
-google-auth==2.3.3
-google-cloud-core==2.2.1
-google-cloud-datastore==2.4.0
-googleapis-common-protos==1.54.0
-protobuf==3.19.1
+idna==3.7
+iniconfig==2.0.0
+itsdangerous==1.1.0
+Jinja2==2.11.3
+jsonschema==4.23.0
+jsonschema-specifications==2023.12.1
 MarkupSafe==2.0.1
-requests==2.26.0
-requests-oauthlib==1.3.0
-configparser==5.2.0
-meilisearch==0.17.0
-Werkzeug==0.16.1
-
-# dev deps
-flake8==4.0.1
+mccabe==0.6.1
+meilisearch==0.31.4
+packaging==24.1
+pluggy==1.5.0
+py==1.11.0
+pycodestyle==2.8.0
+pydantic==2.8.2
+pydantic_core==2.20.1
 pyflakes==2.4.0
 pytest==6.2.5
+pytz==2024.1
+referencing==0.35.1
+requests==2.32.3
+rpds-py==0.20.0
+six==1.16.0
+toml==0.10.2
+typing_extensions==4.12.2
+urllib3==2.2.2
+Werkzeug==0.16.1

docker-compose.yml

@@ -1,67 +1,35 @@
-version: '3'
-
 services:
-  #find a way to tear this down after the init scripts has worked
-  setup-datastore:
-    build: ./scripts/setup-datastore/
-    command: ["/wait-for-it/wait-for-it.sh", "datastore:8000", "--", "make", "run-with-emulator"]
-    depends_on:
-      - datastore
-    environment:
-      MEILISEARCH_HOST: http://meilisearch:7700
-      GOOGLE_APPLICATION_CREDENTIALS: /tmp/key/creds.json
-      GCLOUD_PROJECT: pname
-      DATASTORE_EMULATOR_HOST: datastore:8000
-      DATASTORE_PROJECT_ID: pname
-    volumes:
-      - ./scripts/wait-for-it/:/wait-for-it
-
   #find a way to tear this down after the init scripts has worked
   setup-meilisearch:
     build: ./tools/meilisearch-index-generator/
     command: ["/wait-for-it/wait-for-it.sh", "meilisearch:7700", "--", "make", "run-with-emulator"]
     depends_on:
-      - setup-datastore
       - meilisearch
     environment:
       MEILISEARCH_HOST: http://meilisearch:7700
-      GOOGLE_APPLICATION_CREDENTIALS: /tmp/key/creds.json
-      GCLOUD_PROJECT: pname
-      DATASTORE_EMULATOR_HOST: datastore:8000
-      DATASTORE_PROJECT_ID: pname
+      MEILISEARCH_MASTER_KEY: meilisearch-master-key
     volumes:
       - ./scripts/wait-for-it/:/wait-for-it
 
   meilisearch:
-    image: getmeili/meilisearch
+    image: getmeili/meilisearch:v1.9
     volumes:
       - ./data.ms:/data.ms
+    environment:
+      MEILI_MASTER_KEY: meilisearch-master-key
     ports:
       - 7700:7700
 
-  datastore:
-    build:
-      context: ./gcloud-emulator/
-      dockerfile: Dockerfile-datastore
-    command: gcloud beta emulators datastore start --project=pname --host-port 0.0.0.0:8000 --no-store-on-disk
-    ports:
-      - 8000:8000
-
   api:
     build: ./api
     depends_on:
       - setup-meilisearch
-      - setup-datastore
     environment:
       MEILISEARCH_HOST: http://meilisearch:7700
-      GOOGLE_APPLICATION_CREDENTIALS: /tmp/key/creds.json
-      GCLOUD_PROJECT: pname
-      DATASTORE_EMULATOR_HOST: datastore:8000
-      DATASTORE_PROJECT_ID: pname
+      MEILISEARCH_MASTER_KEY: meilisearch-master-key
     ports:
       - 8811:8811
 
-
   frontend:
     build: ./frontend
     depends_on:
@@ -70,4 +38,4 @@ services:
       REACT_APP_API_BASE_URL: http://localhost:8811
       REACT_ENV: development
     ports:
-      - 3000:3000
+       - 3000:3000