Special character classifier

classifiers/__init__.py

@@ -9,6 +9,10 @@
 
 from .lookup_lists import lookup_list
 
+from .reference_quality import (
+    special_character_classifier,
+)
+
 from .dates_and_times import (
     workday_classifier,
 )
@@ -55,7 +59,8 @@
     distilbert_stock_news_classifier,
     workday_classifier,
     deberta_review_classifier, 
-    bert_sentiment_german
+    bert_sentiment_german,
+    special_character_classifier
 ]:
     module_name = module.__name__.split(".")[-1]
     model_name = (

classifiers/reference_quality/special_character_classifier/README.md

@@ -0,0 +1 @@
+The purpose of this brick is to identify if there are any unusual characters in the given text. This function can be useful for text preprocessing tasks, especially for checking reference material in RAG (Retrieval Augmented Generation) use cases where you want to filter out text that contains unusual or unexpected characters.

classifiers/reference_quality/special_character_classifier/__init__.py

@@ -0,0 +1,37 @@
+import unicodedata
+from typing import Optional, List, Tuple
+from pydantic import BaseModel
+
+INPUT_EXAMPLE = {
+    "text": "Super funny haha 😀.",
+    "allowedRanges": None
+}
+
+ALLOWED_RANGES = set(range(0x0020, 0x007F)).union( # Basic Latin
+    set(range(0x00A0, 0x00FF)), # Latin-1 Supplement
+    set(range(0x0100, 0x017F)),  # Latin Extended-A
+    set(range(0x0180, 0x024F)),  # Latin Extended-B
+    set(range(0x2000, 0x206F)),  # General Punctuation
+    set(range(0x20A0, 0x20CF)),  # Currency Symbols
+    set([ord("\t"), ord("\n"), ord("\r")])# common stop chars
+    )  
+
+class SpecialCharacterClassifierModel(BaseModel):
+    text: str
+    allowed_ranges: Optional[List[Tuple[int,int]]] = None
+
+    class Config:
+        schema_extra = {"example": INPUT_EXAMPLE}
+
+
+def special_character_classifier(req: SpecialCharacterClassifierModel):
+    """Checks if a string contains special characters"""
+    text = req.text
+    allowed_ranges = req.allowed_ranges
+    if allowed_ranges is None:
+        allowed_ranges = ALLOWED_RANGES
+
+    for char in text:
+        if ord(char) not in allowed_ranges and unicodedata.category(char) != "Zs":
+            return {"contains_special_char": "true"}
+    return {"contains_special_char": "false"}

classifiers/reference_quality/special_character_classifier/code_snippet_common.md

@@ -0,0 +1,41 @@
+```python
+import unicodedata
+from typing import List, Tuple
+
+DEFAULT_ALLOWED_RANGES = set(range(0x0020, 0x007F)).union( # Basic Latin
+    set(range(0x00A0, 0x00FF)), # Latin-1 Supplement
+    set(range(0x0100, 0x017F)),  # Latin Extended-A
+    set(range(0x0180, 0x024F)),  # Latin Extended-B
+    set(range(0x2000, 0x206F)),  # General Punctuation
+    set(range(0x20A0, 0x20CF)),  # Currency Symbols
+    set([ord("\t"), ord("\n"), ord("\r")])# common stop chars
+    )  
+
+
+def contains_special_characters(text: str, allowed_ranges: List[Tuple[int,int]] = None) -> str:
+    """
+    @param text: Text to detect special characters in
+    @param allowed_char_codes: Set of allowed char codes.
+    @return: True if text contains unusual characters, False otherwise.
+    """
+
+    if allowed_ranges is None:
+        allowed_ranges = DEFAULT_ALLOWED_RANGES
+
+    for char in text:
+        if ord(char) not in allowed_ranges and unicodedata.category(char) != "Zs":
+            return "true"
+    return "false"
+
+
+# ↑ necessary bricks function 
+# -----------------------------------------------------------------------------------------
+# ↓ example implementation 
+
+def example_integration():
+    texts = ["This contains a special char 你好.", "Such a clean text, wow!", "This is a greek letter: α", "Super funny 😀", "Rainbows are very nice."]
+    for text in texts:
+        print(f"\"{text}\" -> {special_character_classifier(text)}")
+
+example_integration()
+```

classifiers/reference_quality/special_character_classifier/code_snippet_refinery.md

@@ -0,0 +1,27 @@
+```python
+import unicodedata
+from typing import Optional, List, Tuple
+
+ATTRIBUTE: str = "text" # only text attributes
+ALLOWED_RANGES: List[Tuple[int,int]] = None
+
+default_allowed_values = set(range(0x0020, 0x007F)).union( # Basic Latin
+    set(range(0x00A0, 0x00FF)), # Latin-1 Supplement
+    set(range(0x0100, 0x017F)),  # Latin Extended-A
+    set(range(0x0180, 0x024F)),  # Latin Extended-B
+    set(range(0x2000, 0x206F)),  # General Punctuation
+    set(range(0x20A0, 0x20CF)),  # Currency Symbols
+    set([ord("\t"), ord("\n"), ord("\r")])# common stop chars
+)  
+
+def special_character_classifier(record):
+    text = record[ATTRIBUTE].text    
+
+    allowed = ALLOWED_RANGES
+    if not allowed:
+        allowed = default_allowed_values
+    for char in text:
+        if ord(char) not in allowed and unicodedata.category(char) != "Zs":
+            return "true"
+    return "false"
+```

classifiers/reference_quality/special_character_classifier/config.py

@@ -0,0 +1,44 @@
+from util.configs import build_classifier_function_config
+from util.enums import State, BricksVariableType, RefineryDataType, SelectionType
+from . import special_character_classifier, INPUT_EXAMPLE
+
+
+def get_config():
+    return build_classifier_function_config(
+        # strapi information
+        function=special_character_classifier,
+        input_example=INPUT_EXAMPLE,
+        issue_id=345,
+        tabler_icon="LanguageKatakana",
+        min_refinery_version="1.7.0",
+        state=State.PUBLIC.value,
+        type="python_function",
+        available_for=["refinery", "common"],
+        part_of_group=[
+            "reference_quality",
+        ],  # first entry should be parent directory
+        # bricks integrator information
+        cognition_init_mapping = {
+            "true": "Needs fix",
+            "false": "null"
+        },
+        integrator_inputs={
+            "name": "special_character_classifier",
+            "refineryDataType": RefineryDataType.TEXT.value,
+            "variables": {
+                "ATTRIBUTE": {
+                    "selectionType": SelectionType.CHOICE.value,
+                    "addInfo": [
+                        BricksVariableType.ATTRIBUTE.value,
+                        BricksVariableType.GENERIC_STRING.value
+                    ]
+                },
+                "ALLOWED_RANGES": {
+                    "selectionType": SelectionType.LIST.value,
+                    "addInfo": [
+                        BricksVariableType.GENERIC_INT.value
+                    ]
+                }
+            }
+        }      
+    )