alpaca-cot-zh-refine.yaml

# global parameters
project_name: 'Data-Juicer-recipes-alpaca-cot-zh'
dataset_path: '/path/to/your/dataset'  # path to your dataset directory or file
export_path: '/path/to/your/dataset.jsonl'

np: 50  # number of subprocess to process your dataset
open_tracer: true

# process schedule
# a list of several process operators with their arguments
process:
  - document_deduplicator: # 16957516
      lowercase: true # whether to convert text to lower case
      ignore_non_character: true

  - alphanumeric_filter:  # 16957388
      tokenization: false
      min_ratio: 0.10  
  - character_repetition_filter: #  16956845
      rep_len: 10
      max_ratio: 0.6  
  - flagged_words_filter: # 16954629
      lang: zh
      tokenization: true
      use_words_aug: true
      max_ratio: 0.017
  - text_length_filter: # 16954317
      min_len: 10

  - document_simhash_deduplicator: # 9873214
      tokenization: character
      window_size: 4  # small window size for short texts
      lowercase: true
      ignore_pattern: '\p{P}'
      num_blocks: 10
      hamming_distance: 8  # larger hamming distance threshold for short texts