check_number_of_files_and_lines.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## This code is used to go through 9350 folders and read (at least)112,200 .txt files\n",
    "#### I am using to check if the number of files (12) AND if the number of lines (41) on each file is correct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#using glob to get each file path\n",
    "import glob\n",
    "import datetime\n",
    "\n",
    "def check_output_files(base_path,noises,datasets,output_file_name):\n",
    "    #going through each dataset/kfold/noise/score folder\n",
    "    for dataset in datasets:\n",
    "        print(f\"working with dataset: {dataset}\")\n",
    "        log_file = open(output_file_name,\"a\")\n",
    "        for kfold in range(1,11):\n",
    "            for noise in noises:\n",
    "                file_path = base_path+f\"/{dataset}/{kfold:0>3}/{noise}/score/*\"\n",
    "                files = glob.glob(file_path)\n",
    "                total_score_files = int(len(files))\n",
    "                #checking the number of files on this folder\n",
    "                if total_score_files < 12 or total_score_files > 12:\n",
    "                    log_file.write(f\"{dataset}-{kfold:0>3}-{noise}-{total_score_files}_files_of_12_ERROR\\n\")\n",
    "                for file in files:\n",
    "                    line_counter = 0\n",
    "                    #counting the number of lines on this file\n",
    "                    for line in open(file, \"r\"):\n",
    "                        line_counter = line_counter + 1\n",
    "                    if line_counter == 41:\n",
    "                        log_file.write(f\"{dataset}-{kfold:0>3}-{noise}-41_lines_OK\\n\")\n",
    "                    else:\n",
    "                        log_file.write(f\"{dataset}-{kfold:0>3}-{noise}-{line_counter}_lines_ERROR\\n\")\n",
    "        log_file.close()\n",
    "    print(\"finished\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of datasets to be checked:  11\n",
      "working with dataset: friedman1_scores\n",
      "working with dataset: friedman2_scores\n",
      "working with dataset: friedman3_scores\n",
      "working with dataset: peak_d2_scores\n",
      "working with dataset: peak_d3_scores\n",
      "working with dataset: peak_d5_scores\n",
      "working with dataset: peak_d7_scores\n",
      "working with dataset: regression_d2_scores\n",
      "working with dataset: regression_d3_scores\n",
      "working with dataset: regression_d5_scores\n",
      "working with dataset: regression_d7_scores\n",
      "finished\n",
      "Wall time:  0:00:01.597853\n"
     ]
    }
   ],
   "source": [
    "start = datetime.datetime.now()\n",
    "output_file_name = \"check_number_of_files_and_lines_regression.txt\"\n",
    "\n",
    "#path to the datasets folders\n",
    "base_path = \"./regression\"\n",
    "\n",
    "#directories to be explored\n",
    "noises = ['001','002','004','008','016','032','064','128','256','512','1024']\n",
    "\n",
    "#getting datasets folders names into a list\n",
    "datasets_path = glob.glob(base_path+\"/*\")\n",
    "datasets = []\n",
    "for dataset in datasets_path:\n",
    "    datasets.append(dataset.split(\"\\\\\")[1])\n",
    "print(\"Number of datasets to be checked: \", len(noises))\n",
    "\n",
    "#executing the verification\n",
    "check_output_files(base_path,noises,datasets,output_file_name)\n",
    "\n",
    "print(\"Wall time: \", datetime.datetime.now() - start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of datasets to be checked:  11\n",
      "working with dataset: 2dnormals_c2_scores\n",
      "working with dataset: 2dnormals_c3_scores\n",
      "working with dataset: 2dnormals_c5_scores\n",
      "working with dataset: 2dnormals_c7_scores\n",
      "working with dataset: blobs_c2_d2_scores\n",
      "working with dataset: blobs_c2_d3_scores\n",
      "working with dataset: blobs_c2_d5_scores\n",
      "working with dataset: blobs_c2_d7_scores\n",
      "working with dataset: blobs_c3_d2_scores\n",
      "working with dataset: blobs_c3_d3_scores\n",
      "working with dataset: blobs_c3_d5_scores\n",
      "working with dataset: blobs_c3_d7_scores\n",
      "working with dataset: blobs_c5_d2_scores\n",
      "working with dataset: blobs_c5_d3_scores\n",
      "working with dataset: blobs_c5_d5_scores\n",
      "working with dataset: blobs_c5_d7_scores\n",
      "working with dataset: blobs_c7_d2_scores\n",
      "working with dataset: blobs_c7_d3_scores\n",
      "working with dataset: blobs_c7_d5_scores\n",
      "working with dataset: blobs_c7_d7_scores\n",
      "working with dataset: cassini_scores\n",
      "working with dataset: circles_scores\n",
      "working with dataset: circle_d2_scores\n",
      "working with dataset: circle_d3_scores\n",
      "working with dataset: circle_d5_scores\n",
      "working with dataset: circle_d7_scores\n",
      "working with dataset: classification_c2_d2_scores\n",
      "working with dataset: classification_c2_d3_scores\n",
      "working with dataset: classification_c2_d5_scores\n",
      "working with dataset: classification_c2_d7_scores\n",
      "working with dataset: classification_c3_d2_scores\n",
      "working with dataset: classification_c3_d3_scores\n",
      "working with dataset: classification_c3_d5_scores\n",
      "working with dataset: classification_c3_d7_scores\n",
      "working with dataset: classification_c5_d3_scores\n",
      "working with dataset: classification_c5_d5_scores\n",
      "working with dataset: classification_c5_d7_scores\n",
      "working with dataset: classification_c7_d3_scores\n",
      "working with dataset: classification_c7_d5_scores\n",
      "working with dataset: classification_c7_d7_scores\n",
      "working with dataset: cuboids_scores\n",
      "working with dataset: dinisurface_scores\n",
      "working with dataset: gaussian_quantiles_c2_d2_scores\n",
      "working with dataset: gaussian_quantiles_c2_d3_scores\n",
      "working with dataset: gaussian_quantiles_c2_d5_scores\n",
      "working with dataset: gaussian_quantiles_c2_d7_scores\n",
      "working with dataset: gaussian_quantiles_c3_d2_scores\n",
      "working with dataset: gaussian_quantiles_c3_d3_scores\n",
      "working with dataset: gaussian_quantiles_c3_d5_scores\n",
      "working with dataset: gaussian_quantiles_c3_d7_scores\n",
      "working with dataset: gaussian_quantiles_c5_d2_scores\n",
      "working with dataset: gaussian_quantiles_c5_d3_scores\n",
      "working with dataset: gaussian_quantiles_c5_d5_scores\n",
      "working with dataset: gaussian_quantiles_c5_d7_scores\n",
      "working with dataset: gaussian_quantiles_c7_d2_scores\n",
      "working with dataset: gaussian_quantiles_c7_d3_scores\n",
      "working with dataset: gaussian_quantiles_c7_d5_scores\n",
      "working with dataset: gaussian_quantiles_c7_d7_scores\n",
      "working with dataset: helicoid_scores\n",
      "working with dataset: hypercube_d2_scores\n",
      "working with dataset: hypercube_d3_scores\n",
      "working with dataset: moons_scores\n",
      "working with dataset: ringnorm_d2_scores\n",
      "working with dataset: ringnorm_d3_scores\n",
      "working with dataset: ringnorm_d5_scores\n",
      "working with dataset: ringnorm_d7_scores\n",
      "working with dataset: shapes_scores\n",
      "working with dataset: smiley_scores\n",
      "working with dataset: spirals_c2_scores\n",
      "working with dataset: spirals_c3_scores\n",
      "working with dataset: spirals_c5_scores\n",
      "working with dataset: spirals_c7_scores\n",
      "working with dataset: spirals_scores\n",
      "working with dataset: swissroll_scores\n",
      "working with dataset: threenorm_d2_scores\n",
      "working with dataset: threenorm_d3_scores\n",
      "working with dataset: threenorm_d5_scores\n",
      "working with dataset: threenorm_d7_scores\n",
      "working with dataset: twonorm_d2_scores\n",
      "working with dataset: twonorm_d3_scores\n",
      "working with dataset: twonorm_d5_scores\n",
      "working with dataset: twonorm_d7_scores\n",
      "working with dataset: waveform_scores\n",
      "working with dataset: xor_d2_scores\n",
      "working with dataset: xor_d3_scores\n",
      "finished\n",
      "Wall time:  0:38:28.029769\n"
     ]
    }
   ],
   "source": [
    "start = datetime.datetime.now()\n",
    "output_file_name = \"check_number_of_files_and_lines_classification.txt\"\n",
    "\n",
    "#path to the datasets folders\n",
    "base_path = \"./classification\"\n",
    "\n",
    "#directories to be explored\n",
    "noises = ['001','002','004','008','016','032','064','128','256','512','1024']\n",
    "\n",
    "#getting datasets folders names into a list\n",
    "datasets_path = glob.glob(base_path+\"/*\")\n",
    "datasets = []\n",
    "for dataset in datasets_path:\n",
    "    datasets.append(dataset.split(\"\\\\\")[1])\n",
    "print(\"Number of datasets to be checked: \", len(noises))\n",
    "\n",
    "#executing the verification\n",
    "check_output_files(base_path,noises,datasets,output_file_name)\n",
    "\n",
    "print(\"Wall time: \", datetime.datetime.now() - start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}