diff --git a/assistant_dists/dream_multimodal/cpu.yml b/assistant_dists/dream_multimodal/cpu.yml index d073539763..4347af0c4d 100644 --- a/assistant_dists/dream_multimodal/cpu.yml +++ b/assistant_dists/dream_multimodal/cpu.yml @@ -12,3 +12,7 @@ services: environment: DEVICE: cpu CUDA_VISIBLE_DEVICES: "" + fromage: + environment: + DEVICE: cpu + CUDA_VISIBLE_DEVICES: "" diff --git a/assistant_dists/dream_multimodal/dev.yml b/assistant_dists/dream_multimodal/dev.yml index 9697392b85..36c4987a36 100644 --- a/assistant_dists/dream_multimodal/dev.yml +++ b/assistant_dists/dream_multimodal/dev.yml @@ -10,23 +10,23 @@ services: - 3000:3000 volumes: - "~/.deeppavlov/file_server:/tmp" - dff-program-y-skill: - volumes: - - "./skills/dff_program_y_skill:/src" - - "./common:/src/common" - ports: - - 8008:8008 sentseg: volumes: - "./annotators/SentSeg:/src" ports: - 8011:8011 - convers-evaluation-selector: + ranking-based-response-selector: volumes: - - "./response_selectors/convers_evaluation_based_selector:/src" + - "./response_selectors/ranking_based_response_selector:/src" + - "./common:/src/common" + ports: + - 8002:8002 + dff-program-y-skill: + volumes: + - "./skills/dff_program_y_skill:/src" - "./common:/src/common" ports: - - 8009:8009 + - 8008:8008 dff-intent-responder-skill: volumes: - "./skills/dff_intent_responder_skill:/src" @@ -46,11 +46,6 @@ services: - "./common:/src/common" ports: - 8018:8018 - spelling-preprocessing: - volumes: - - "./annotators/spelling_preprocessing:/src" - ports: - - 8074:8074 dialogpt: volumes: - "./services/dialogpt:/src" @@ -76,4 +71,17 @@ services: - "./common:/src/common" ports: - 8124:8124 + fromage: + volumes: + - "./services/fromage:/src" + - "./common:/src/common" + - "~/.deeppavlov/cache:/root/.cache" + ports: + - 8069:8069 + dff-fromage-image-skill: + volumes: + - "./skills/dff_fromage_image_skill:/src" + - "./common:/src/common" + ports: + - 8070:8070 version: "3.7" diff --git a/assistant_dists/dream_multimodal/docker-compose.override.yml b/assistant_dists/dream_multimodal/docker-compose.override.yml index 737527c8b8..93784edd04 100644 --- a/assistant_dists/dream_multimodal/docker-compose.override.yml +++ b/assistant_dists/dream_multimodal/docker-compose.override.yml @@ -2,9 +2,10 @@ services: agent: command: sh -c 'bin/wait && python -m deeppavlov_agent.run agent.pipeline_config=assistant_dists/dream_multimodal/pipeline_conf.json' environment: - WAIT_HOSTS: "dff-program-y-skill:8008, sentseg:8011, convers-evaluation-selector:8009, + WAIT_HOSTS: "dff-program-y-skill:8008, sentseg:8011, ranking-based-response-selector:8002, dff-intent-responder-skill:8012, intent-catcher:8014, badlisted-words:8018, - spelling-preprocessing:8074, dialogpt:8125, sentence-ranker:8128, image-captioning:8123, dff-image-skill:8124" + dialogpt:8125, sentence-ranker:8128, image-captioning:8123, dff-image-skill:8124, + fromage:8069, dff-fromage-image-skill:8070" WAIT_HOSTS_TIMEOUT: ${WAIT_TIMEOUT:-1200} HIGH_PRIORITY_INTENTS: 1 RESTRICTION_FOR_SENSITIVE_CASE: 1 @@ -15,26 +16,33 @@ services: files: image: julienmeerschart/simple-file-upload-download-server - dff-program-y-skill: - env_file: [.env] + ranking-based-response-selector: + env_file: [ .env ] build: args: - SERVICE_PORT: 8008 - SERVICE_NAME: dff_program_y_skill + SERVICE_PORT: 8002 + SERVICE_NAME: response_selector LANGUAGE: EN + SENTENCE_RANKER_ANNOTATION_NAME: sentence_ranker + SENTENCE_RANKER_SERVICE_URL: http://sentence-ranker:8128/respond + SENTENCE_RANKER_TIMEOUT: 3 + N_UTTERANCES_CONTEXT: 5 + FILTER_TOXIC_OR_BADLISTED: 1 + FALLBACK_FILE: fallbacks_dream_en.json context: . - dockerfile: ./skills/dff_program_y_skill/Dockerfile - command: gunicorn --workers=1 server:app -b 0.0.0.0:8008 --reload + dockerfile: ./response_selectors/ranking_based_response_selector/Dockerfile + command: flask run -h 0.0.0.0 -p 8002 + environment: + - FLASK_APP=server deploy: resources: limits: - memory: 1024M + memory: 100M reservations: - memory: 1024M - + memory: 100M sentseg: - env_file: [.env] + env_file: [ .env ] build: context: ./annotators/SentSeg/ command: flask run -h 0.0.0.0 -p 8011 @@ -47,41 +55,22 @@ services: reservations: memory: 1.5G - convers-evaluation-selector: + dff-program-y-skill: env_file: [.env] build: args: - TAG_BASED_SELECTION: 1 - CALL_BY_NAME_PROBABILITY: 0.5 - PROMPT_PROBA: 0.1 - ACKNOWLEDGEMENT_PROBA: 0.3 - PRIORITIZE_WITH_REQUIRED_ACT: 0 - PRIORITIZE_NO_DIALOG_BREAKDOWN: 0 - PRIORITIZE_WITH_SAME_TOPIC_ENTITY: 0 - IGNORE_DISLIKED_SKILLS: 0 - GREETING_FIRST: 1 - RESTRICTION_FOR_SENSITIVE_CASE: 1 - PRIORITIZE_PROMTS_WHEN_NO_SCRIPTS: 1 - MAX_TURNS_WITHOUT_SCRIPTS: 7 - ADD_ACKNOWLEDGMENTS_IF_POSSIBLE: 1 - PRIORITIZE_SCRIPTED_SKILLS: 0 - CONFIDENCE_STRENGTH: 0.8 - CONV_EVAL_STRENGTH: 0.4 - PRIORITIZE_HUMAN_INITIATIVE: 1 - QUESTION_TO_QUESTION_DOWNSCORE_COEF: 0.8 + SERVICE_PORT: 8008 + SERVICE_NAME: dff_program_y_skill LANGUAGE: EN - FALLBACK_FILE: fallbacks_dream_en.json context: . - dockerfile: ./response_selectors/convers_evaluation_based_selector/Dockerfile - command: flask run -h 0.0.0.0 -p 8009 - environment: - - FLASK_APP=server + dockerfile: ./skills/dff_program_y_skill/Dockerfile + command: gunicorn --workers=1 server:app -b 0.0.0.0:8008 --reload deploy: resources: limits: - memory: 256M + memory: 1024M reservations: - memory: 256M + memory: 1024M dff-intent-responder-skill: env_file: [ .env ] @@ -137,23 +126,6 @@ services: reservations: memory: 256M - spelling-preprocessing: - env_file: [ .env ] - build: - args: - SERVICE_PORT: 8074 - SERVICE_NAME: spelling_preprocessing - context: ./annotators/spelling_preprocessing/ - command: flask run -h 0.0.0.0 -p 8074 - environment: - - FLASK_APP=server - deploy: - resources: - limits: - memory: 100M - reservations: - memory: 100M - dialogpt: env_file: [ .env ] build: @@ -230,5 +202,40 @@ services: memory: 1024M reservations: memory: 1024M - -version: '3.7' + + fromage: + env_file: [ .env ] + build: + args: + SERVICE_PORT: 8069 + SERVICE_NAME: fromage + RET_SCALE_FACTOR: 0 + context: . + dockerfile: ./services/fromage/Dockerfile + command: flask run -h 0.0.0.0 -p 8069 + environment: + - CUDA_VISIBLE_DEVICES=0 + - FLASK_APP=server + deploy: + resources: + limits: + memory: 45G + reservations: + memory: 45G + + dff-fromage-image-skill: + env_file: [.env] + build: + args: + SERVICE_PORT: 8070 + SERVICE_NAME: dff_fromage_image_skill + context: . + dockerfile: ./skills/dff_fromage_image_skill/Dockerfile + command: gunicorn --workers=1 server:app -b 0.0.0.0:8070 --reload + deploy: + resources: + limits: + memory: 100M + reservations: + memory: 100M +version: '3.7' \ No newline at end of file diff --git a/assistant_dists/dream_multimodal/pipeline_conf.json b/assistant_dists/dream_multimodal/pipeline_conf.json index dd121ae995..9fb41a6e49 100644 --- a/assistant_dists/dream_multimodal/pipeline_conf.json +++ b/assistant_dists/dream_multimodal/pipeline_conf.json @@ -58,22 +58,6 @@ } }, "annotators": { - "spelling_preprocessing": { - "connector": { - "protocol": "http", - "timeout": 1.0, - "url": "http://spelling-preprocessing:8074/respond" - }, - "dialog_formatter": "state_formatters.dp_formatters:last_utt_dialog", - "response_formatter": "state_formatters.dp_formatters:simple_formatter_service", - "previous_services": [], - "state_manager_method": "add_annotation_and_reset_human_attributes_for_first_turn", - "is_enabled": true, - "source": { - "component": "components/pGxj32ic41pvquRXUdqc7A.yml", - "service": "annotators/spelling_preprocessing/service_configs/spelling-preprocessing" - } - }, "sentseg": { "connector": { "protocol": "http", @@ -82,9 +66,7 @@ }, "dialog_formatter": "state_formatters.dp_formatters:preproc_last_human_utt_dialog", "response_formatter": "state_formatters.dp_formatters:simple_formatter_service", - "previous_services": [ - "annotators.spelling_preprocessing" - ], + "previous_services": [], "state_manager_method": "add_annotation", "is_enabled": true, "source": { @@ -100,9 +82,7 @@ }, "dialog_formatter": "state_formatters.dp_formatters:preproc_last_human_utt_dialog", "response_formatter": "state_formatters.dp_formatters:simple_formatter_service", - "previous_services": [ - "annotators.spelling_preprocessing" - ], + "previous_services": [], "state_manager_method": "add_annotation", "is_enabled": true, "source": { @@ -119,7 +99,6 @@ "dialog_formatter": "state_formatters.dp_formatters:last_utt_sentseg_segments_dialog", "response_formatter": "state_formatters.dp_formatters:simple_formatter_service", "previous_services": [ - "annotators.spelling_preprocessing", "annotators.sentseg" ], "state_manager_method": "add_annotation", @@ -143,6 +122,21 @@ "component": "components/f2hECzXDTTfljm5mweoMFA.yml", "service": "services/image_captioning/service_configs/image-captioning" } + }, + "fromage": { + "connector": { + "protocol": "http", + "timeout": 90.0, + "url": "http://fromage:8069/respond" + }, + "dialog_formatter": "state_formatters.dp_formatters:fromage_formatter", + "response_formatter": "state_formatters.dp_formatters:simple_formatter_service", + "state_manager_method": "add_annotation", + "is_enabled": true, + "source": { + "component": "components/8iHHdjsnfhewkl.yml", + "service": "services/fromage/service_configs/fromage" + } } }, "response_annotators": { @@ -314,6 +308,24 @@ "service": "services/dialogpt/service_configs/dialogpt" } }, + "dff_fromage_image_skill": { + "connector": { + "protocol": "http", + "timeout": 2.0, + "url": "http://dff-fromage-image-skill:8070/respond" + }, + "dialog_formatter": "state_formatters.dp_formatters:dff_fromage_image_skill_formatter", + "response_formatter": "state_formatters.dp_formatters:skill_with_attributes_formatter_service", + "previous_services": [ + "skill_selectors" + ], + "state_manager_method": "add_hypothesis", + "is_enabled": true, + "source": { + "component": "components/8jfFjmYnbdeH.yml", + "service": "services/dff_fromage_image_skill/service_configs/dff-fromage-image-skill" + } + }, "dff_image_skill": { "connector": { "protocol": "http", @@ -338,9 +350,9 @@ "connector": { "protocol": "http", "timeout": 1.0, - "url": "http://convers-evaluation-selector:8009/respond" + "url": "http://ranking-based-response-selector:8002/respond" }, - "dialog_formatter": "state_formatters.dp_formatters:full_history_dialog", + "dialog_formatter": "state_formatters.dp_formatters:cropped_dialog", "response_formatter": "state_formatters.dp_formatters:base_response_selector_formatter_service", "previous_services": [ "candidate_annotators" @@ -348,8 +360,8 @@ "state_manager_method": "add_bot_utterance", "is_enabled": true, "source": { - "component": "components/ly2AVNtIcJpTWz1qJ1mvKQ.yml", - "service": "response_selectors/convers_evaluation_based_selector/service_configs/convers-evaluation-selector" + "component": "components/YJzc7NwGrLmKp6gfZJh7X1.yml", + "service": "response_selectors/ranking_based_response_selector/service_configs/ranking-based-response-selector" } } } diff --git a/assistant_dists/dream_multimodal/proxy.yml b/assistant_dists/dream_multimodal/proxy.yml index e494238b2d..4e99cfed89 100644 --- a/assistant_dists/dream_multimodal/proxy.yml +++ b/assistant_dists/dream_multimodal/proxy.yml @@ -35,5 +35,4 @@ services: environment: - PROXY_PASS=proxy.deeppavlov.ai:8128 - PORT=8128 - version: '3.7' diff --git a/assistant_dists/dream_multimodal/test.yml b/assistant_dists/dream_multimodal/test.yml new file mode 100644 index 0000000000..c9d72b9222 --- /dev/null +++ b/assistant_dists/dream_multimodal/test.yml @@ -0,0 +1,52 @@ +services: + agent: + volumes: + - "/cephfs/home/ignatov/artifacts:/output" + ports: + - ${AGENT_PORT}:4242 + mongo: + command: mongod + image: mongo:4.0.0 + files: + volumes: + - "~/.deeppavlov/file_server:/tmp" + ranking-based-response-selector: + volumes: + - "./response_selectors/ranking_based_response_selector:/src" + - "./common:/src/common" + sentseg: + dff-intent-responder-skill: + volumes: + - "~/.deeppavlov:/root/.deeppavlov" + - "~/.deeppavlov/cache:/root/.cache" + dff-fromage-image-skill: + volumes: + - "~/.deeppavlov:/root/.deeppavlov" + - "~/.deeppavlov/cache:/root/.cache" + intent-catcher: + volumes: + - "~/.deeppavlov:/root/.deeppavlov" + - "~/.deeppavlov/cache:/root/.cache" + badlisted-words: + volumes: + - "~/.deeppavlov:/root/.deeppavlov" + - "~/.deeppavlov/cache:/root/.cache" + fromage: + volumes: + - "~/.deeppavlov:/root/.deeppavlov" + - "~/.deeppavlov/cache:/root/.cache" + environment: + - CUDA_VISIBLE_DEVICES=0 + sentence-ranker: + volumes: + - "~/.deeppavlov:/root/.deeppavlov" + - "~/.deeppavlov/cache:/root/.cache" + image-captioning: + volumes: + - "~/.deeppavlov:/root/.deeppavlov" + - "~/.deeppavlov/cache:/root/.cache" + dff-image-skill: + volumes: + - "~/.deeppavlov:/root/.deeppavlov" + - "~/.deeppavlov/cache:/root/.cache" +version: "3.7" \ No newline at end of file diff --git a/assistant_dists/dream_multimodal/test_4.yml b/assistant_dists/dream_multimodal/test_4.yml new file mode 100644 index 0000000000..616b4375d9 --- /dev/null +++ b/assistant_dists/dream_multimodal/test_4.yml @@ -0,0 +1,52 @@ +services: + agent: + volumes: + - "/cephfs/home/ignatov/artifacts:/output" + ports: + - ${AGENT_PORT}:4242 + mongo: + command: mongod + image: mongo:4.0.0 + files: + volumes: + - "~/.deeppavlov/file_server:/tmp" + ranking-based-response-selector: + volumes: + - "./response_selectors/ranking_based_response_selector:/src" + - "./common:/src/common" + sentseg: + dff-intent-responder-skill: + volumes: + - "~/.deeppavlov:/root/.deeppavlov" + - "~/.deeppavlov/cache:/root/.cache" + dff-fromage-image-skill: + volumes: + - "~/.deeppavlov:/root/.deeppavlov" + - "~/.deeppavlov/cache:/root/.cache" + intent-catcher: + volumes: + - "~/.deeppavlov:/root/.deeppavlov" + - "~/.deeppavlov/cache:/root/.cache" + badlisted-words: + volumes: + - "~/.deeppavlov:/root/.deeppavlov" + - "~/.deeppavlov/cache:/root/.cache" + fromage: + volumes: + - "~/.deeppavlov:/root/.deeppavlov" + - "~/.deeppavlov/cache:/root/.cache" + environment: + - CUDA_VISIBLE_DEVICES=0,1,2,3 + sentence-ranker: + volumes: + - "~/.deeppavlov:/root/.deeppavlov" + - "~/.deeppavlov/cache:/root/.cache" + image-captioning: + volumes: + - "~/.deeppavlov:/root/.deeppavlov" + - "~/.deeppavlov/cache:/root/.cache" + dff-image-skill: + volumes: + - "~/.deeppavlov:/root/.deeppavlov" + - "~/.deeppavlov/cache:/root/.cache" +version: "3.7" \ No newline at end of file diff --git a/components.tsv b/components.tsv index 57dc68ea06..4c852ac0a4 100644 --- a/components.tsv +++ b/components.tsv @@ -70,8 +70,8 @@ 8066 news-api-skill 8067 8068 game-cooperative-skill -8069 -8070 +8069 fromage +8070 dff-fromage-image-skill 8071 factoid-qa 8072 kbqa 8073 diff --git a/components/8iHHdjsnfhewkl.yml b/components/8iHHdjsnfhewkl.yml new file mode 100644 index 0000000000..b6d5a71310 --- /dev/null +++ b/components/8iHHdjsnfhewkl.yml @@ -0,0 +1,23 @@ +name: fromage +display_name: FROMAGe Service +component_type: null +model_type: NN-based +is_customizable: false +author: publisher@deeppavlov.ai +description: The service is built using the FROMAGe model, which is able to produce meaningful conversations with users about different images. +ram_usage: 45G +gpu_usage: 20G +group: skills +connector: + protocol: http + timeout: 90.0 + url: http://fromage:8069/respond +dialog_formatter: state_formatters.dp_formatters:fromage_formatter +response_formatter: state_formatters.dp_formatters:simple_formatter_service +previous_services: null +required_previous_services: null +state_manager_method: add_annotation +tags: null +endpoint: respond +service: services/fromage/service_configs/fromage +date_created: '2023-03-16T09:45:32' \ No newline at end of file diff --git a/components/8jfFjmYnbdeH.yml b/components/8jfFjmYnbdeH.yml new file mode 100644 index 0000000000..b5d572ed08 --- /dev/null +++ b/components/8jfFjmYnbdeH.yml @@ -0,0 +1,24 @@ +name: dff_fromage_image_skill +display_name: Fromage Image Skill +component_type: Script-based w/o NNs +model_type: Dictionary/Pattern-based +is_customizable: false +author: publisher@deeppavlov.ai +description: A skill that utilizes information from the FROMAGe service and engages in a dialogue with the user about an image. +ram_usage: 100M +gpu_usage: null +group: skills +connector: + protocol: http + timeout: 2.0 + url: http://dff-fromage-image-skill:8070/respond +dialog_formatter: state_formatters.dp_formatters:dff_fromage_image_skill_formatter +response_formatter: state_formatters.dp_formatters:skill_with_attributes_formatter_service +previous_services: +- skill_selectors +required_previous_services: null +state_manager_method: add_hypothesis +tags: null +endpoint: respond +service: skills/dff_fromage_image_skill/service_configs/dff-fromage-image-skill +date_created: '2023-03-16T09:45:32' \ No newline at end of file diff --git a/services/agent_services/service_configs/dream_multimodal/environment.yml b/services/agent_services/service_configs/dream_multimodal/environment.yml index cf3ef6ea90..4de7832710 100644 --- a/services/agent_services/service_configs/dream_multimodal/environment.yml +++ b/services/agent_services/service_configs/dream_multimodal/environment.yml @@ -1,5 +1,8 @@ -WAIT_HOSTS: '' -WAIT_HOSTS_TIMEOUT: ${WAIT_TIMEOUT:-480} +WAIT_HOSTS: sentseg:8011, badlisted-words:8018, intent-catcher:8014, + image-captioning:8123, fromage:8069, dff-program-y-skill:8008, dff-intent-responder-skill:8012, + dff-fromage-image-skill:8070, dff-image-skill:8124, ranking-based-response-selector:8002, dialogpt:8125, + sentence-ranker:8128 +WAIT_HOSTS_TIMEOUT: ${WAIT_TIMEOUT:-1200} HIGH_PRIORITY_INTENTS: 1 RESTRICTION_FOR_SENSITIVE_CASE: 1 ALWAYS_TURN_ON_ALL_SKILLS: 0 diff --git a/services/fromage/Dockerfile b/services/fromage/Dockerfile new file mode 100644 index 0000000000..d2ba082f31 --- /dev/null +++ b/services/fromage/Dockerfile @@ -0,0 +1,36 @@ +# syntax=docker/dockerfile:experimental + +FROM pytorch/pytorch:1.5-cuda10.1-cudnn7-runtime + +WORKDIR /src + +ARG PRETRAINED_MODEL_FNAME +ENV PRETRAINED_MODEL_FNAME ${PRETRAINED_MODEL_FNAME} +ARG CONFIG_NAME +ENV CONFIG_NAME ${CONFIG_NAME} +ARG SERVICE_PORT +ENV SERVICE_PORT ${SERVICE_PORT} +ARG RET_SCALE_FACTOR +ENV RET_SCALE_FACTOR ${RET_SCALE_FACTOR} + + +ENV PYTHONPATH "/src/fromage:/fromage:$PYTHONPATH" + +COPY ./services/fromage/requirements.txt /src/requirements.txt +RUN pip install -r /src/requirements.txt + +RUN apt-get update && apt-get install git -y +RUN pip install gdown==4.7.1 + +RUN mkdir /fromage && \ + git clone https://github.com/ciwwwnd/fromage.git /fromage + +RUN mkdir -p /services/fromage/fromage_model +RUN gdown 1wMojZNqEwApNlsCZVvSgQVtZLgbeLoKi -O /services/fromage/fromage_model/cc3m_embeddings.pkl +RUN gdown 1qyDiUw6uMA4nijLaNpr3J-2pigdIANYE -O /services/fromage/fromage_model/model_args.json +RUN gdown 1oG_fWDje3M6XBoU2GtrOlrqaffJEhxyN -O /services/fromage/fromage_model/pretrained_ckpt.pth.tar + +COPY ./services/fromage/ /src/ +COPY ./common/ ./common/ + +CMD gunicorn --workers=1 server:app -b 0.0.0.0:${SERVICE_PORT} --timeout=1200 diff --git a/services/fromage/README.md b/services/fromage/README.md new file mode 100644 index 0000000000..b0007ab923 --- /dev/null +++ b/services/fromage/README.md @@ -0,0 +1,16 @@ +# FROMAGe Service +**FROMAGe** is a service that is used to process an input image and respond to the user's questions accordingly. It is based on the [FROMAGe](https://github.com/kohjingyu/fromage/tree/main) model from [Grounding Language Models to Images for Multimodal Inputs and Outputs](https://arxiv.org/abs/2301.13823). + +GPU RAM 20 GB, RAM 45 GB. + +## Running server + +```sh +sudo AGENT_PORT=4242 docker-compose -f docker-compose.yml -f assistant_dists/dream_multimodal/docker-compose.override.yml -f assistant_dists/dream_multimodal/dev.yml -f assistant_dists/dream_multimodal/test.yml up --build fromage +``` + +## Testing + +```sh +./test.sh +``` diff --git a/services/fromage/requirements.txt b/services/fromage/requirements.txt new file mode 100644 index 0000000000..ffee96d138 --- /dev/null +++ b/services/fromage/requirements.txt @@ -0,0 +1,47 @@ +flask==1.1.1 +itsdangerous==2.0.1 +fromage==1.1.0 +gunicorn==19.9.0 +requests==2.22.0 +sentry-sdk[flask]==0.14.1 +healthcheck==1.3.3 +jinja2<=3.0.3 +Werkzeug<=2.0.3 +attrs==22.2.0 +certifi==2022.12.7 +charset-normalizer==3.0.1 +contourpy==1.0.6 +cycler==0.11.0 +einops==0.4.1 +exceptiongroup==1.1.0 +filelock==3.9.0 +fonttools==4.38.0 +huggingface-hub==0.12.0 +idna==2.8 +iniconfig==2.0.0 +kiwisolver==1.4.4 +matplotlib==3.5.3 +numpy==1.21.6 +packaging==23.0 +pandas==1.3.5 +Pillow==9.4.0 +pluggy==1.0.0 +pyparsing==3.0.9 +pytest==7.2.1 +python-dateutil==2.8.2 +regex==2022.10.31 +six==1.16.0 +tensorboard==2.11.2 +tensorboard-data-server==0.6.0 +tensorboard-plugin-wit==1.8.1 +tokenizers==0.12.1 +tomli==2.0.1 +torch==1.13.1 +torchaudio==0.11.0 +torchmetrics==0.9.3 +torchvision==0.12.0 +tqdm==4.64.1 +transformers==4.21.3 +typing_extensions==4.4.0 +urllib3==1.21.1 +gitpython==3.1.31 diff --git a/services/fromage/server.py b/services/fromage/server.py new file mode 100644 index 0000000000..ceca363359 --- /dev/null +++ b/services/fromage/server.py @@ -0,0 +1,71 @@ +import logging +import os +import time +from fromage import models +from fromage import utils +import torch +import sentry_sdk +from flask import Flask, request, jsonify +from sentry_sdk.integrations.flask import FlaskIntegration + +sentry_sdk.init(dsn=os.getenv("SENTRY_DSN"), integrations=[FlaskIntegration()]) + +logging.basicConfig(format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", level=logging.INFO) +logger = logging.getLogger(__name__) + +FILE_SERVER_URL = os.getenv("FILE_SERVER_URL") +RET_SCALE_FACTOR = int(os.environ.get("RET_SCALE_FACTOR")) + + +try: + model_dir = "/services/fromage/fromage_model" + model = models.load_fromage(model_dir) + + if torch.cuda.is_available(): + logger.info("fromage is set to run on cuda") + + logger.info("fromage is ready") +except Exception as e: + sentry_sdk.capture_exception(e) + logger.exception(e) + raise e + +app = Flask(__name__) +logging.getLogger("werkzeug").setLevel("WARNING") + + +def generate_responses(image_path, prompt): + inp_image = [utils.get_image_from_url(image_path)] + if prompt == "": + prompt = ["What is the image?"] + text = "" + for p in prompt: + text += f"Q: {p}\nA:" + model_prompt = inp_image + [text] + model_outputs = model.generate_for_images_and_texts( + model_prompt, num_words=32, ret_scale_factor=RET_SCALE_FACTOR, max_num_rets=0 + ) + text += " ".join([s for s in model_outputs if isinstance(s, str)]) + "\n" + return model_outputs + + +@app.route("/respond", methods=["POST"]) +def respond(): + st_time = time.time() + image_paths = request.json.get("image_paths", []) + sentences = request.json.get("sentences", []) + + try: + frmg_answers = [] + for image_path, sentence in zip(image_paths, sentences): + outputs = generate_responses(image_path, sentence) + frmg_answers += outputs + except Exception as exc: + logger.exception(exc) + sentry_sdk.capture_exception(exc) + frmg_answers = [[""]] * len(sentences) + + total_time = time.time() - st_time + logger.info(f"fromage results: {frmg_answers}") + logger.info(f"fromage exec time: {total_time:.3f}s") + return jsonify(frmg_answers) diff --git a/services/fromage/service_configs/fromage/environment.yml b/services/fromage/service_configs/fromage/environment.yml new file mode 100644 index 0000000000..f715424aaa --- /dev/null +++ b/services/fromage/service_configs/fromage/environment.yml @@ -0,0 +1,4 @@ +SERVICE_PORT: 8069 +SERVICE_NAME: fromage +FLASK_APP: server +RET_SCALE_FACTOR: 0 diff --git a/services/fromage/service_configs/fromage/service.yml b/services/fromage/service_configs/fromage/service.yml new file mode 100644 index 0000000000..ff1dab6794 --- /dev/null +++ b/services/fromage/service_configs/fromage/service.yml @@ -0,0 +1,28 @@ +name: fromage +endpoints: +- respond +compose: + env_file: + - .env + build: + args: + SERVICE_PORT: 8069 + SERVICE_NAME: fromage + FLASK_APP: server + RET_SCALE_FACTOR: 0 + context: . + dockerfile: ./services/fromage/Dockerfile + command: flask run -h 0.0.0.0 -p 8069 + environment: + - FLASK_APP=server + deploy: + resources: + limits: + memory: 45G + reservations: + memory: 45G + volumes: + - ./services/fromage:/src + - ./common:/src/common + ports: + - 8069:8069 diff --git a/services/fromage/test.py b/services/fromage/test.py new file mode 100644 index 0000000000..1812e80adb --- /dev/null +++ b/services/fromage/test.py @@ -0,0 +1,19 @@ +import requests + + +def test_respond(): + url = "http://0.0.0.0:8069/respond" + + image_paths = ["https://s0.rbk.ru/v6_top_pics/media/img/7/26/346832135841267.jpg"] + sentences = ["What is the make of the car?"] + request_data = {"image_paths": image_paths, "sentences": sentences} + result = requests.post(url, json=request_data).json() + print(result) + + obligatory_word = "SUV" + assert obligatory_word in result[0], f"Expected the word '{obligatory_word}' to present in caption" + print("\n", "Success!!!") + + +if __name__ == "__main__": + test_respond() diff --git a/services/fromage/test.sh b/services/fromage/test.sh new file mode 100755 index 0000000000..468a5a38fc --- /dev/null +++ b/services/fromage/test.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +python test.py \ No newline at end of file diff --git a/services/image_captioning/Dockerfile b/services/image_captioning/Dockerfile index 51ee32cc45..c23b7af539 100644 --- a/services/image_captioning/Dockerfile +++ b/services/image_captioning/Dockerfile @@ -18,7 +18,8 @@ RUN conda remove PyYAML WORKDIR /ofa -RUN git clone https://github.com/dariamitciuk/OFA.git && \ +RUN git clone https://github.com/dariamitciuk/OFA.git /ofa && \ + cd /ofa && \ pip install -r requirements.txt && \ git checkout 153048138044edcbe0b099463810a971a7bf0057 diff --git a/skill_selectors/rule_based_selector/connector.py b/skill_selectors/rule_based_selector/connector.py index 5d7e166297..b40ef4d07e 100644 --- a/skill_selectors/rule_based_selector/connector.py +++ b/skill_selectors/rule_based_selector/connector.py @@ -97,6 +97,9 @@ async def send(self, payload: Dict, callback: Callable): dialog_len = len(dialog["human_utterances"]) if user_uttr.get("attributes", {}).get("image") is not None: skills_for_uttr.append("dff_image_skill") + if any("image" in user_uttr.get("attributes", {}) for user_uttr in dialog["human_utterances"][-5:]): + skills_for_uttr.append("dff_fromage_image_skill") + exit_cond = "exit" in intent_catcher_intents and ( dialog_len == 1 or (dialog_len == 2 and len(user_uttr_text.split()) > 3) ) diff --git a/skills/dff_fromage_image_skill/Dockerfile b/skills/dff_fromage_image_skill/Dockerfile new file mode 100644 index 0000000000..998fc6c9f8 --- /dev/null +++ b/skills/dff_fromage_image_skill/Dockerfile @@ -0,0 +1,29 @@ +FROM python:3.9.1 +# ###################### IMMUTABLE SECTION ###################################### +# Do not change anything in this section +WORKDIR /src + +COPY common/dff/requirements.txt . +RUN pip install -r requirements.txt + +# ###################### CUSTOM SECTION ###################################### +# Here you can make changes + +ARG SERVICE_NAME +ENV SERVICE_NAME ${SERVICE_NAME} + +COPY skills/${SERVICE_NAME}/requirements.txt . +RUN pip install -r requirements.txt +RUN python -m nltk.downloader wordnet + +COPY skills/${SERVICE_NAME}/ ./ +COPY ./common/ ./common/ + +ARG SERVICE_PORT +ENV SERVICE_PORT ${SERVICE_PORT} + +# wait for a server answer ( INTERVAL + TIMEOUT ) * RETRIES seconds after that change stutus to unhealthy +HEALTHCHECK --interval=5s --timeout=5s --retries=3 CMD curl --fail 127.0.0.1:${SERVICE_PORT}/healthcheck || exit 1 + + +CMD gunicorn --workers=1 server:app -b 0.0.0.0:${SERVICE_PORT} diff --git a/skills/dff_fromage_image_skill/README.md b/skills/dff_fromage_image_skill/README.md new file mode 100644 index 0000000000..5982184cbe --- /dev/null +++ b/skills/dff_fromage_image_skill/README.md @@ -0,0 +1,31 @@ +# dff-fromage-image-skill + +## Description + +**dff-fromage-image-skill** is a simple service that can discuss images + +## Quickstart from docker + +```bash +# create local.yml +python utils/create_local_yml.py -s dff-fromage-image-skill +# build service +docker-compose -f docker-compose.yml -f local.yml up -d --build dff-fromage-image-skill +# run tests +docker-compose -f docker-compose.yml -f local.yml exec dff-fromage-image-skill bash test.sh +# run a dialog with the agent +docker-compose -f docker-compose.yml -f local.yml exec agent python -m deeppavlov_agent.run +``` + +## Quickstart without docker + +```bash +pip install -r requirements.txt +gunicorn --workers=1 server:app -b 0.0.0.0:${SERVICE_PORT} +``` + +## Resources + +* Execution time: 46 ms +* Starting time: 1.5 sec +* RAM: 1024 MB diff --git a/skills/dff_fromage_image_skill/common/.gitkeep b/skills/dff_fromage_image_skill/common/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/skills/dff_fromage_image_skill/requirements.txt b/skills/dff_fromage_image_skill/requirements.txt new file mode 100644 index 0000000000..500562e428 --- /dev/null +++ b/skills/dff_fromage_image_skill/requirements.txt @@ -0,0 +1,6 @@ +click==7.1.2 +nltk==3.5 +requests==2.27.1 +Pillow==9.1.0 +fastapi==0.73.0 +uvicorn==0.17.4 \ No newline at end of file diff --git a/skills/dff_fromage_image_skill/scenario/condition.py b/skills/dff_fromage_image_skill/scenario/condition.py new file mode 100644 index 0000000000..f44a8cf126 --- /dev/null +++ b/skills/dff_fromage_image_skill/scenario/condition.py @@ -0,0 +1,14 @@ +import logging + +from df_engine.core import Context, Actor +import common.dff.integration.context as int_ctx + +logger = logging.getLogger(__name__) +logger.setLevel(logging.NOTSET) + + +def caption_condition(ctx: Context, actor: Actor, *args, **kwargs) -> bool: + caption = int_ctx.get_last_human_utterance(ctx, actor).get("annotations", {}).get("fromage", {}) + if caption: + return True + return False diff --git a/skills/dff_fromage_image_skill/scenario/main.py b/skills/dff_fromage_image_skill/scenario/main.py new file mode 100644 index 0000000000..c8daca69ee --- /dev/null +++ b/skills/dff_fromage_image_skill/scenario/main.py @@ -0,0 +1,44 @@ +import logging + +from df_engine.core.keywords import ( + TRANSITIONS, + GLOBAL, + RESPONSE, +) +from df_engine.core import Actor +from . import condition as loc_cnd +from . import response as loc_rsp + +logger = logging.getLogger(__name__) + +flows = { + GLOBAL: { + TRANSITIONS: { + ("fromage_caption_response", "general_node"): loc_cnd.caption_condition, + } + }, + "fromage_caption_response": { + "general_node": { + RESPONSE: loc_rsp.generic_response, + TRANSITIONS: {}, + }, + }, + "global_flow": { + "start": { + RESPONSE: "", + TRANSITIONS: {}, + }, + "fallback": { + RESPONSE: "Okay. Why did you send me this picture?", + TRANSITIONS: {}, + }, + }, +} + +actor = Actor( + flows, + start_label=("global_flow", "start"), + fallback_label=("global_flow", "fallback"), +) + +logger.info("Actor created successfully") diff --git a/skills/dff_fromage_image_skill/scenario/response.py b/skills/dff_fromage_image_skill/scenario/response.py new file mode 100644 index 0000000000..a898bdf818 --- /dev/null +++ b/skills/dff_fromage_image_skill/scenario/response.py @@ -0,0 +1,16 @@ +import logging +from df_engine.core import Context, Actor +import common.dff.integration.context as int_ctx + +logger = logging.getLogger(__name__) + +DEFAULT_CONFIDENCE = 0.95 +SUPER_CONFIDENCE = 1 + + +def generic_response(ctx: Context, actor: Actor, excluded_skills=None, *args, **kwargs) -> str: + caption = int_ctx.get_last_human_utterance(ctx, actor).get("annotations", {}).get("fromage", {}) + + logger.debug(f"fromage image skill {caption}") + int_ctx.set_confidence(ctx, actor, DEFAULT_CONFIDENCE) + return caption diff --git a/skills/dff_fromage_image_skill/server.py b/skills/dff_fromage_image_skill/server.py new file mode 100644 index 0000000000..a8eeed22d7 --- /dev/null +++ b/skills/dff_fromage_image_skill/server.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python + +import logging +import time +import os +import random + +from flask import Flask, request, jsonify +from healthcheck import HealthCheck +import sentry_sdk +from sentry_sdk.integrations.logging import ignore_logger + + +from common.dff.integration.actor import load_ctxs, get_response + +from scenario.main import actor + +import test_server + + +ignore_logger("root") + +sentry_sdk.init(os.getenv("SENTRY_DSN")) +SERVICE_NAME = os.getenv("SERVICE_NAME") +SERVICE_PORT = int(os.getenv("SERVICE_PORT")) +RANDOM_SEED = int(os.getenv("RANDOM_SEED", 2718)) + +logging.basicConfig(format="%(asctime)s - %(pathname)s - %(lineno)d - %(levelname)s - %(message)s", level=logging.DEBUG) +logger = logging.getLogger(__name__) + + +app = Flask(__name__) +health = HealthCheck(app, "/healthcheck") +logging.getLogger("werkzeug").setLevel("WARNING") + + +def handler(requested_data, random_seed=None): + st_time = time.time() + ctxs = load_ctxs(requested_data) + random_seed = requested_data.get("random_seed", random_seed) # for tests + + responses = [] + for ctx in ctxs: + try: + # for tests + if random_seed: + random.seed(int(random_seed)) + ctx = actor(ctx) + responses.append(get_response(ctx, actor)) + except Exception as exc: + sentry_sdk.capture_exception(exc) + logger.exception(exc) + responses.append(("", 1.0, {}, {}, {})) + + total_time = time.time() - st_time + logger.info(f"{SERVICE_NAME} exec time = {total_time:.3f}s") + return responses + + +try: + test_server.run_test(handler) + logger.info("test query processed") +except Exception as exc: + sentry_sdk.capture_exception(exc) + logger.exception(exc) + raise exc + +logger.info(f"{SERVICE_NAME} is loaded and ready") + +# import pathlib +# import json + +# for in_file in pathlib.Path("tests").glob("./*_in.json"): +# logger.error(in_file) +# test_in = json.load(in_file.open()) +# responses = handler(test_in, RANDOM_SEED) +# out_file = str(in_file).replace("in.json", "out.json") +# import common.test_utils as t_utils + +# t_utils.save_to_test(responses, out_file, indent=4) # TEST + + +@app.route("/respond", methods=["POST"]) +def respond(): + # import common.test_utils as t_utils; t_utils.save_to_test(request.json,"tests/lets_talk_in.json",indent=4) # TEST + # responses = handler(request.json, RANDOM_SEED) # TEST + # import common.test_utils as t_utils; t_utils.save_to_test(responses,"tests/lets_talk_out.json",indent=4) # TEST + responses = handler(request.json) + return jsonify(responses) + + +if __name__ == "__main__": + app.run(debug=False, host="0.0.0.0", port=SERVICE_PORT) diff --git a/skills/dff_fromage_image_skill/service_configs/dff-fromage-image-skill/environment.yml b/skills/dff_fromage_image_skill/service_configs/dff-fromage-image-skill/environment.yml new file mode 100644 index 0000000000..9c14320667 --- /dev/null +++ b/skills/dff_fromage_image_skill/service_configs/dff-fromage-image-skill/environment.yml @@ -0,0 +1,2 @@ +SERVICE_PORT: 8070 +SERVICE_NAME: dff_fromage_image_skill \ No newline at end of file diff --git a/skills/dff_fromage_image_skill/service_configs/dff-fromage-image-skill/service.yml b/skills/dff_fromage_image_skill/service_configs/dff-fromage-image-skill/service.yml new file mode 100644 index 0000000000..4060e3d001 --- /dev/null +++ b/skills/dff_fromage_image_skill/service_configs/dff-fromage-image-skill/service.yml @@ -0,0 +1,24 @@ +name: dff-fromage-image-skill +endpoints: +- respond +compose: + env_file: + - .env + build: + args: + SERVICE_PORT: 8070 + SERVICE_NAME: dff_fromage_image_skill + context: . + dockerfile: ./skills/dff_fromage_image_skill/Dockerfile + command: gunicorn --workers=1 server:app -b 0.0.0.0:8070 + deploy: + resources: + limits: + memory: 100M + reservations: + memory: 100M + volumes: + - ./skills/dff_fromage_image_skill:/src + - ./common:/src/common + ports: + - 8070:8070 diff --git a/skills/dff_fromage_image_skill/test.sh b/skills/dff_fromage_image_skill/test.sh new file mode 100644 index 0000000000..f85ff6a382 --- /dev/null +++ b/skills/dff_fromage_image_skill/test.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +python test_server.py diff --git a/skills/dff_fromage_image_skill/test_server.py b/skills/dff_fromage_image_skill/test_server.py new file mode 100644 index 0000000000..5ceb78f9ef --- /dev/null +++ b/skills/dff_fromage_image_skill/test_server.py @@ -0,0 +1,33 @@ +import requests +import os + +import common.test_utils as test_utils + + +SERVICE_PORT = int(os.getenv("SERVICE_PORT")) +RANDOM_SEED = int(os.getenv("RANDOM_SEED", 2718)) +URL = f"http://0.0.0.0:{SERVICE_PORT}/respond" + + +def handler(requested_data, random_seed): + hypothesis = requests.post(URL, json={**requested_data, "random_seed": random_seed}).json() + return hypothesis + + +def run_test(handler): + in_data, out_data = test_utils.get_dataset() + for test_name in in_data: + hypothesis = handler(in_data[test_name], RANDOM_SEED) + print(f"test name: {test_name}") + is_equal_flag, msg = test_utils.compare_structs(out_data[test_name], hypothesis, ignored_keys=["id"]) + if msg and len(msg.split("`")) == 5: + _, ground_truth_text, _, hypothesis_text, _ = msg.split("`") + is_equal_flag, ratio = test_utils.compare_text(ground_truth_text, hypothesis_text, 0.0) + if not is_equal_flag: + msg = f"{msg} ratio = {ratio}" + assert is_equal_flag, msg + print("Success") + + +if __name__ == "__main__": + run_test(handler) diff --git a/skills/dff_fromage_image_skill/tests/.gitkeep b/skills/dff_fromage_image_skill/tests/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/state_formatters/dp_formatters.py b/state_formatters/dp_formatters.py index 1707c1690a..110a4a6f39 100755 --- a/state_formatters/dp_formatters.py +++ b/state_formatters/dp_formatters.py @@ -1056,6 +1056,29 @@ def dff_image_skill_formatter(dialog: Dict) -> List[Dict]: return utils.dff_formatter(dialog, "dff_image_skill") +def dff_fromage_image_skill_formatter(dialog: Dict) -> List[Dict]: + return utils.dff_formatter(dialog, "dff_fromage_image_skill") + + +def fromage_formatter(dialog: Dict) -> List: + # Used by: fromage + dialog = utils.get_last_n_turns(dialog) + dialog = utils.remove_clarification_turns_from_dialog(dialog) + + image_paths = [utt["attributes"].get("image") for utt in dialog["human_utterances"]] + utterances_history = 5 + image_paths = image_paths[-utterances_history:] + human_text_uttr = dialog["human_utterances"][-1]["text"] + input_dict = {"sentences": [human_text_uttr if human_text_uttr else ""]} + for url in reversed(image_paths): + if url is not None and url.startswith("http"): + input_dict.update({"image_paths": [url]}) + break + else: + input_dict.update({"image_paths": [None]}) + return [input_dict] + + def dff_prompted_skill_formatter(dialog, skill_name=None): return utils.dff_formatter( dialog,