Fixed issues related to path and duplication

Fixed issues related to path and duplication

.dockerignore.

@@ -0,0 +1,2 @@
+# Ignore files and directories
+.history

Dockerfile

@@ -6,6 +6,7 @@ FROM golang:buster AS builder
 ARG JDK_VERSION=11
 
 RUN apt-get update && \ 
+    apt-get install ca-certificates-java openjdk-${JDK_VERSION}-jre-headless -y && \
     apt-get install -y --no-install-recommends openjdk-${JDK_VERSION}-jdk && \
     apt-get install aapt -y && \
     apt-get clean && \

apk/analysis.go

@@ -16,12 +16,13 @@ limitations under the License.
 
 import (
 	"encoding/json"
-	"io/ioutil"
+	"fmt"
 	database "morf/db"
 	"morf/models"
+	"morf/utils"
 	util "morf/utils"
 	"net/http"
-	"os"
+	"path/filepath"
 
 	log "github.com/sirupsen/logrus"
 
@@ -31,6 +32,9 @@ import (
 )
 
 func StartCliExtraction(apkPath string, db *gorm.DB, is_db_req bool) {
+	var fileName string
+
+	fs := utils.GetAppFS()
 	if is_db_req {
 		apkFound, json_data := util.CheckDuplicateInDB(db, apkPath)
 		if apkFound {
@@ -40,35 +44,47 @@ func StartCliExtraction(apkPath string, db *gorm.DB, is_db_req bool) {
 	}
 	packageModel := ExtractPackageData(apkPath)
 	metadata := StartMetaDataCollection(apkPath)
-	scanner_data := StartSecScan("temp/input/" + apkPath)
+
+	fmt.Println("Metadata: Completed")
+
+	if apkPath[0] == '/' {
+		fileName = filepath.Base(apkPath)
+	} else {
+		fileName = apkPath
+	}
+
+	scanner_data := StartSecScan(utils.GetInputDir() + fileName)
+	scanner_data = utils.SanitizeSecrets(scanner_data)
 	secret_data, secret_error := json.Marshal(scanner_data)
 
 	if secret_error != nil {
 		log.Error(secret_error)
 	}
 
-	secret := util.CreateSecretModel(apkPath, packageModel, metadata, scanner_data, secret_data)
+	secret := util.CreateSecretModel(fileName, packageModel, metadata, scanner_data, secret_data)
+
 	if is_db_req {
 		database.InsertSecrets(secret, db)
 	}
+
 	json_data, json_error := json.MarshalIndent(secret, "", " ")
 
 	if json_error != nil {
 		log.Error(json_error)
 	}
 
-	_, err_ := os.Stat(vip.GetString("backup_path"))
-
-	if os.IsNotExist(err_) {
-		os.Mkdir(vip.GetString("backup_path"), 0755)
+	//Check if backup folder exists
+	if !util.CheckBackUpDirExists(fs) {
+		util.CreateBackUpDir(fs)
 	}
 
-	err := ioutil.WriteFile(vip.GetString("backup_path")+"/"+apkPath+"_"+secret.APKVersion+".json", json_data, 0644)
-	if err != nil {
-		log.Error(err)
-	}
+	util.WriteToFile(fs, vip.GetString("backup_path")+fileName+"_"+secret.APKVersion+".json", string(json_data))
+	util.WriteToFile(fs, vip.GetString("backup_path")+fileName+"_"+"Secrets_"+secret.APKVersion+".json", string(secret_data))
 
-	log.Info("APK Data saved to: " + vip.GetString("backup_path") + "/" + apkPath + "_" + secret.APKVersion + ".json")
+	util.WriteToFile(fs, "results"+"/"+fileName+"_"+secret.APKVersion+".json", string(json_data))
+	util.WriteToFile(fs, "results"+"/"+fileName+"_"+"Secrets_"+secret.APKVersion+".json", string(secret_data))
+
+	log.Info("APK Data saved to: " + vip.GetString("backup_path") + "/" + fileName + "_" + secret.APKVersion + ".json")
 }
 
 func StartJiraProcess(jiramodel models.JiraModel, db *gorm.DB, c *gin.Context) {
@@ -92,7 +108,7 @@ func StartJiraProcess(jiramodel models.JiraModel, db *gorm.DB, c *gin.Context) {
 
 	packageModel := ExtractPackageData(apk_path)
 	metadata := StartMetaDataCollection(apk_path)
-	scanner_data := StartSecScan("temp/input/" + apk_path)
+	scanner_data := StartSecScan(utils.GetInputDir() + apk_path)
 	secret_data, secret_error := json.Marshal(scanner_data)
 
 	if secret_error != nil {
@@ -106,6 +122,7 @@ func StartJiraProcess(jiramodel models.JiraModel, db *gorm.DB, c *gin.Context) {
 }
 
 func StartExtractProcess(apkPath string, db *gorm.DB, c *gin.Context, isSlack bool, slackData models.SlackData) {
+	fs := utils.GetAppFS()
 
 	apkFound, json_data := util.CheckDuplicateInDB(db, apkPath)
 	if apkFound {
@@ -124,15 +141,14 @@ func StartExtractProcess(apkPath string, db *gorm.DB, c *gin.Context, isSlack bo
 
 	packageModel := ExtractPackageData(apkPath)
 	metadata := StartMetaDataCollection(apkPath)
-	scanner_data := StartSecScan("temp/input/" + apkPath)
+	scanner_data := StartSecScan(utils.GetInputDir() + apkPath)
 	secret_data, secret_error := json.Marshal(scanner_data)
 
 	if secret_error != nil {
 		log.Error(secret_error)
 	}
 
 	secret := util.CreateSecretModel(apkPath, packageModel, metadata, scanner_data, secret_data)
-
 	database.InsertSecrets(secret, db)
 
 	json_data, json_error := json.MarshalIndent(secret, "", " ")
@@ -143,22 +159,16 @@ func StartExtractProcess(apkPath string, db *gorm.DB, c *gin.Context, isSlack bo
 	}
 
 	//Check if backup folder exists
-	_, err_ := os.Stat(vip.GetString("backup_path"))
-
-	if os.IsNotExist(err_) {
-		os.Mkdir(vip.GetString("backup_path"), 0755)
+	//Check if backup folder exists
+	if !util.CheckBackUpDirExists(fs) {
+		util.CreateBackUpDir(fs)
 	}
 
-	// Check if file exists
+	util.WriteToFile(fs, vip.GetString("backup_path")+apkPath+"_"+secret.APKVersion+".json", string(json_data))
+	util.WriteToFile(fs, vip.GetString("backup_path")+apkPath+"_"+"Secrets_"+secret.APKVersion+".json", string(secret_data))
 
-	//Move the APK Data to backup folder
-	backupPath := vip.GetString("backup_path") + apkPath + "_" + secret.APKVersion + ".json"
-	log.Println("Backup Path: ", backupPath)
-	err := ioutil.WriteFile(backupPath, json_data, 0644)
-
-	if err != nil {
-		log.Error(err)
-	}
+	util.WriteToFile(fs, "results"+"/"+apkPath+"_"+secret.APKVersion+".json", string(json_data))
+	util.WriteToFile(fs, "results"+"/"+apkPath+"_"+"Secrets_"+secret.APKVersion+".json", string(secret_data))
 
 	if !isSlack {
 		c.JSON(http.StatusOK, gin.H{

apk/metadata.go

@@ -20,38 +20,41 @@ import (
 	"io"
 	"log"
 	"morf/models"
+	"morf/utils"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"strings"
+
+	alf "github.com/spf13/afero"
 )
 
 func StartMetaDataCollection(apkPath string) models.MetaDataModel {
 	// Check if temp directory exist and If yes delete it and create a new one
 
-	if _, err := os.Stat("temp"); err == nil {
+	fs := alf.NewOsFs()
+
+	if utils.CheckifmorftmpDirExists(fs) {
 		fmt.Println("Deleting the temp directory")
-		os.RemoveAll("temp")
+		utils.DeleteTmpDir(fs)
 		fmt.Println("Creating a new temp directory")
-		os.Mkdir("temp", 0777)
+		utils.CreateMorfDirintmp(fs)
 	} else {
 		fmt.Println("Creating a new temp directory")
-		os.Mkdir("temp", 0777)
+		utils.CreateMorfDirintmp(fs)
 	}
 
-	if _, err := os.Stat("temp/input"); os.IsNotExist(err) {
-		os.Mkdir("temp/input", 0755)
-	}
-	if _, err := os.Stat("temp/output"); os.IsNotExist(err) {
-		os.Mkdir("temp/output", 0755)
+	// Create input and output directory
+	if _, err := os.Stat(utils.GetInputDir()); os.IsNotExist(err) {
+		utils.CreateInputOutputDir(fs)
 	}
 
-	os.Rename(apkPath, "temp/input/"+apkPath)
-	apkPath = "temp/input/" + apkPath
+	// Move APK to input directory
+
+	apkPath = utils.CopyApktoInputDir(fs, apkPath)
 	fmt.Println("Starting metadata collection for " + apkPath)
 
-	metadata_success, metadata_error := exec.Command("java", "-cp", "tools/apkanalyzer.jar", "sk.styk.martin.bakalarka.execute.Main", "-analyze", "--in", "temp/input/", "--out", "temp/output").Output()
-	fmt.Println(metadata_success)
+	metadata_success, metadata_error := exec.Command("java", "-cp", "tools/apkanalyzer.jar", "sk.styk.martin.bakalarka.execute.Main", "-analyze", "--in", utils.GetInputDir(), "--out", utils.GetOutputDir()).Output()
 
 	if metadata_error != nil {
 		fmt.Println("Error while decompiling the APK file")
@@ -65,8 +68,8 @@ func StartMetaDataCollection(apkPath string) models.MetaDataModel {
 		fmt.Println(file_path)
 
 		// Make file readable
-		os.Chmod("temp/output/"+strings.Replace(file_name, ".apk", ".json", -1), 0777)
-		return startFileParser("temp/output/" + strings.Replace(file_name, ".apk", ".json", -1))
+		os.Chmod(utils.GetOutputDir()+strings.Replace(file_name, ".apk", ".json", -1), 0777)
+		return startFileParser(utils.GetOutputDir() + strings.Replace(file_name, ".apk", ".json", -1))
 	}
 
 	return models.MetaDataModel{}

apk/scanner.go

@@ -19,7 +19,6 @@ package apk
 
 import (
 	"fmt"
-	"io/ioutil"
 	"morf/models"
 	"morf/utils"
 	"os"
@@ -60,56 +59,43 @@ func CheckAPK(apkPath string) {
 
 func StartSecScan(apkPath string) []models.SecretModel {
 	//Decompile the sources of the APK file
-	counter := 0
 
+	counter := 0
 	log.Println("Decompiling the APK file for sources")
-	source_decompile, source_error := exec.Command("java", "-jar", "tools/apktool.jar", "d", "-r", apkPath, "-o", "temp/output/apk/source").Output()
-
-	if source_error != nil {
-		log.Println("Error while decompiling the APK file")
-		log.Fatal(source_error)
-	}
+	fmt.Println(apkPath)
+	source_decompile, source_error := exec.Command("java", "-jar", "tools/apktool.jar", "d", "-r", apkPath, "-o", utils.GetSourceDir()).Output()
+	utils.HandleError(source_error, "Error while decompiling the APK file", true)
 
 	if source_decompile != nil {
 		log.Println("Decompiling the APK file for sources successful")
 		counter++
 	}
 
 	//Decompile the resources of the APK file
-
-	res_decompile, res_error := utils.ExecuteCommand("java", []string{"-jar", "tools/apktool.jar", "d", "-s", apkPath, "-o", "temp/output/apk/appreso"}, false, true)
-
-	if res_error != nil {
-		log.Println("Error while decompiling the resources of the APK file")
-		log.Error(res_error)
-	}
+	res_decompile, res_error := utils.ExecuteCommand("java", []string{"-jar", "tools/apktool.jar", "d", "-s", apkPath, "-o", utils.GetResDir()}, false, true)
+	utils.HandleError(res_error, "Error while decompiling the APK file", true)
 
 	if res_decompile != nil {
 		log.Println("Decompiling the APK file for resources successful")
 		counter++
 	}
-	files_path := "temp/output/apk/"
+
 	if counter == 2 {
 		log.Println("Decompiling the APK file successful")
-		return StartScan(files_path)
+		return StartScan(utils.GetFilesDir())
 	}
+
 	return nil
 }
 
-func readPatternFile(patternFilePath string) ([]byte, error) {
-	patternFile, err := os.OpenFile(patternFilePath, os.O_RDONLY, 0666)
-	defer patternFile.Close()
-	utils.HandleError(err, "Error opening pattern file:", true)
+func readPatternFile(patternFilePath string) []byte {
 
-	yamlFile, err := ioutil.ReadAll(patternFile)
-	utils.HandleError(err, "Error reading pattern file:", true)
-
-	return yamlFile, err
+	yamlFile := utils.ReadFile(utils.GetAppFS(), patternFilePath)
+	return yamlFile
 }
 
 func StartScan(apkPath string) []models.SecretModel {
-	files, err := ioutil.ReadDir("patterns")
-	utils.HandleError(err, "Error reading directory:", true)
+	files := utils.ReadDir(utils.GetAppFS(), "patterns")
 
 	var wg sync.WaitGroup
 	resultsChan := make(chan models.SecretModel, 100)
@@ -122,9 +108,9 @@ func StartScan(apkPath string) []models.SecretModel {
 			wg.Add(1)
 			go func(file os.FileInfo) {
 				defer wg.Done()
-				yamlFile, err := readPatternFile("patterns/" + file.Name())
+				yamlFile := readPatternFile("patterns/" + file.Name())
 				// Make sure file name is ending with .yml or .yaml
-
+				err := error(nil)
 				if err != nil {
 					fmt.Println(err)
 				}
@@ -147,7 +133,6 @@ func StartScan(apkPath string) []models.SecretModel {
 
 				for _, pattern := range secretPatterns.Patterns {
 					pat := pattern.Pattern.Regex
-					fmt.Println(pat)
 					stdout, err := utils.ExecuteCommand("rg", []string{"-n", "-e", fmt.Sprintf("\"%s\"", pat), "--multiline", apkPath}, true, false)
 
 					utils.HandleError(err, "Error running ripgrep:", true)

cmd/cli.go

@@ -15,6 +15,7 @@ limitations under the License.
 */package cmd
 
 import (
+	"fmt"
 	"morf/apk"
 	"morf/db"
 
@@ -65,6 +66,8 @@ func add(cmd *cob.Command, args []string) {
 			return
 		}
 	}
+	fmt.Println("IS DB REQ", is_db_req)
+	// Check if APK path is absolute or relative
 
 	apk.StartCliExtraction(apkPath, db.DB, is_db_req)
 }

cmd/root.go

@@ -12,7 +12,7 @@ distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
-*//*
+*/ /*
 Copyright © 2023 NAME HERE <EMAIL ADDRESS>
 */
 package cmd
@@ -72,7 +72,6 @@ func runMORF(cmd *cob.Command, args []string) error {
 func init() {
 
 	vip.SetDefault("port", 8080)
-	vip.SetDefault("tempPath", "/temp")
 	vip.SetDefault("backup_path", "backup/")
 
 	MorfCmd.AddCommand(cliCmd)

log/infoLog.log

@@ -1,5 +1,5 @@
-16:40:13,951  INFO ApkProcessingTask:51 -  [CRED-4.0.9-RC0-qa.apk] Started processing of file CRED-4.0.9-RC0-qa.apk
-   16:40:13,955 TRACE FileInfoProcessor:56 -  [CRED-4.0.9-RC0-qa.apk] Started processing of file info
-   16:40:13,957 TRACE FileInfoProcessor:64 -  [CRED-4.0.9-RC0-qa.apk] Finished processing of file info
-   16:40:13,963 TRACE CertificateProcessor:94 -  [CRED-4.0.9-RC0-qa.apk] Started processing of certificate
+12:11:19,293  INFO ApkProcessingTask:51 -  [cred-4.2.3-rc0-release.apk] Started processing of file cred-4.2.3-rc0-release.apk
+   12:11:19,295 TRACE FileInfoProcessor:56 -  [cred-4.2.3-rc0-release.apk] Started processing of file info
+   12:11:19,297 TRACE FileInfoProcessor:64 -  [cred-4.2.3-rc0-release.apk] Finished processing of file info
+   12:11:19,306 TRACE CertificateProcessor:94 -  [cred-4.2.3-rc0-release.apk] Started processing of certificate