restore and download speed up for ubuntu image

app/build.gradle.kts

@@ -130,7 +130,7 @@ dependencies {
     annotationProcessor("com.github.bumptech.glide:compiler:4.11.0")
     implementation("com.github.bachors:Android-Img2Ascii:1.1")
     implementation("org.rauschig:jarchivelib:0.8.0")
-    implementation("org.apache.commons:commons-compress:1.20")
+//    implementation("org.apache.commons:commons-compress:1.26.2")
     implementation("com.github.mwiede:jsch:0.2.11")
 
     implementation("com.github.yuriy-budiyev:code-scanner:2.3.2")
@@ -145,8 +145,10 @@ dependencies {
 
     implementation(platform("org.jetbrains.kotlin:kotlin-bom"))
     implementation("org.jetbrains.kotlin:kotlin-stdlib")
-    implementation("io.ktor:ktor-server-netty:1.4.3")
+//    implementation("io.ktor:ktor-server-netty:1.4.3")
     implementation("ch.qos.logback:logback-classic:1.2.3")
+
+
     testImplementation("org.jetbrains.kotlin:kotlin-test")
     testImplementation("org.jetbrains.kotlin:kotlin-test-junit")
 

app/src/main/assets/ubuntu_setup/support/setup_rootfs.sh

@@ -4,7 +4,13 @@
 # It is assumed that you are root user.
 # sudo su -
 
-export backup_dir_path="${APP_ROOT_PATH}/ubuntu/backup/temp" \
+apt-get autoremove \
+;dpkg -l 'linux-*' | sed '/^ii/!d;/'"$(uname -r | sed "s/\(.*\)-\([^0-9]\+\)/\1/")"'/d;s/^[^ ]* [^ ]* \([^ ]*\).*/\1/;/[0-9]/!d' | xargs apt-get -y purge \
+;apt-get clean \
+;sudo pip cache purge \
+;sudo pip3 cache purge \
+;export APP_ROOT_PATH="/storage/emulated/0/Documents/cmdclick" \
+;export backup_dir_path="${APP_ROOT_PATH}/ubuntu/backup/temp" \
 ;export rootfsTarGz="rootfs.tar.gz" \
 ; cd / \
 ; apt-get purge \

app/src/main/assets/ubuntu_setup/support/startup.sh

@@ -5,6 +5,7 @@ readonly SUPPORT_DIR_PATH="/support"
 readonly USR_LOCAL_BIN="/usr/local/bin"
 readonly UBUNTU_SETUP_COMP_FILE="${SUPPORT_DIR_PATH}/ubuntuSetupComp.txt"
 readonly UBUNTU_LAUNCH_COMP_FILE="${SUPPORT_DIR_PATH}/ubuntuLaunchComp.txt"
+readonly FIRST_SETUP_OK_FILE="${SUPPORT_DIR_PATH}/firstSetupOk.txt"
 
 kill_front_and_sub_process(){
 	local killId=$(\
@@ -210,7 +211,9 @@ setup_dropbear_sshserver(){
 startup_launch_system(){
 	echo "### $FUNCNAME"
 	rm -rf  /tmp/pulse* &
-
+	echo --- launch sshd server
+	echo "DROPBEAR_SSH_PORT ${DROPBEAR_SSH_PORT}"
+	dropbear -E -p ${DROPBEAR_SSH_PORT} >/dev/null &
 	su - "${CMDCLICK_USER}" <<-EOF
 	export APP_ROOT_PATH="${APP_ROOT_PATH}"
 	export MONITOR_DIR_PATH="${MONITOR_DIR_PATH}"
@@ -223,7 +226,7 @@ startup_launch_system(){
 	echo \$USER
 	echo --- launch sshd server
 	echo "DROPBEAR_SSH_PORT ${DROPBEAR_SSH_PORT}"
-	sudo dropbear -E -p ${DROPBEAR_SSH_PORT} >/dev/null &
+	# sudo dropbear -E -p ${DROPBEAR_SSH_PORT} >/dev/null &
 	# 2>&1 &
 	echo "Type bellow command"
 	echo -e "\tssh -p ${DROPBEAR_SSH_PORT}  cmdclick@{your android ip_address}"
@@ -297,20 +300,34 @@ install_require_pacakges(){
 
 install_httpshd(){
 	echo "### ${FUNCNAME}"
-	local package_name="httpshd"
-	chmod +x "/usr/local/bin/${package_name}"
+    local package_name="httpshd"
+    curl \
+        -sSL "https://github.com/puutaro/httpshd/releases/download/0.0.1/httpshd-0.0.1-arm64" \
+        > "${package_name}"
+    local usrlocalbin_httpshd="/usr/local/bin/${package_name}"
+    mv -vf \
+        "${package_name}" \
+        "/usr/local/bin/${package_name}"
+    chmod +x "${usrlocalbin_httpshd}"
 }
 
 install_repbash(){
 	echo "### ${FUNCNAME}"
-	local package_name="repbash"
-	chmod +x "/usr/local/bin/${package_name}"
-	echo "install ok"
+    local package_name="repbash"
+    curl \
+        -sSL "https://github.com/puutaro/repbash/releases/download/0.0.1/repbash-0.0.1-arm64" \
+        > "${package_name}"
+    local usrlocalbin_repbash="/usr/local/bin/${package_name}"
+    mv -vf \
+        "${package_name}" \
+        "${usrlocalbin_repbash}"
+    chmod +x "${usrlocalbin_repbash}"
 }
 
 install_bin(){
-	install_httpshd
-	install_repbash
+	install_httpshd &
+	install_repbash &
+	wait
 }
 
 
@@ -320,6 +337,7 @@ install_base_pkg(){
 	install_require_pacakges
 	install_pip3_pkg webssh
 	install_pip3_pkg yt-dlp
+	install_bin
 	# install_shell2http
 }
 
@@ -341,11 +359,16 @@ install_user_package(){
 	EOF
 }
 
-launch_setup(){
+
+first_setup(){
+	if [ -f "${FIRST_SETUP_OK_FILE}" ];then
+		return
+	fi
 	echo "### ${FUNCNAME}"
+	install_bin &
 	apt-get purge --auto-remove -y sudo
 	apt-get install -y sudo
-	install_bin
+	touch "${FIRST_SETUP_OK_FILE}"
 }
 
 echo_ubuntu_env_tsv_con(){
@@ -399,14 +422,15 @@ if [ ! -f "${UBUNTU_SETUP_COMP_FILE}" ] \
 	setup_dropbear_sshserver
 fi
 if [ ! -f "${UBUNTU_SETUP_COMP_FILE}" ];then \
-	launch_setup
+	# launch_setup
 	touch "${UBUNTU_SETUP_COMP_FILE}"
 fi
 kill_front_and_sub_process
 startup_launch_system &
 launch_extra_startup &
 put_launch_comp_file \
 	"${ubuntu_env_tsv_con}"
+first_setup
 wait
 exit 0
 install_golang_and_go_package

app/src/main/assets/ubuntu_setup/support/ubuntu_manager/backup.sh

@@ -3,38 +3,103 @@
 exec repbash "${0}" \
 	-t "\${UBUNTU_ENV_TSV_PATH}"
 
-readonly UBUNTU_BACKUP_DIR_PATH="$(dirname "${UBUNTU_BACKUP_ROOTFS_PATH}")"
-readonly ROOTFS_TAR_GZ="$(basename "${UBUNTU_BACKUP_ROOTFS_PATH}")"
-readonly UBUNUT_BACKUP_TMP_DIR_PATH="$(dirname "${UBUNTU_BACKUP_TEMP_ROOTFS_PATH}")"
-
 echo "extract.." >> "${MONITOR_FILE_PATH}"
-mkdir -p "${UBUNUT_BACKUP_TMP_DIR_PATH}"
-rm -f "${UBUNTU_BACKUP_TEMP_ROOTFS_PATH}"
+sudo apt-get clean
+sudo apt-get autoremove
+sudo pip cache purge
+sudo pip3 cache purge
+dpkg -l 'linux-*' | sed '/^ii/!d;/'"$(uname -r | sed "s/\(.*\)-\([^0-9]\+\)/\1/")"'/d;s/^[^ ]* [^ ]* \([^ ]*\).*/\1/;/[0-9]/!d' | xargs sudo apt-get -y purge
+mkdir -p "${UBUNTU_BACKUP_TEMP_ROOTFS_DIR_PATH}"
 cd / 
-sudo tar \
-	-cvpzf "${UBUNTU_BACKUP_TEMP_ROOTFS_PATH}" \
-	--exclude=/sys \
-	--exclude=/dev \
-	--exclude=/proc \
-	--exclude=/data \
-	--exclude=/mnt \
-	--exclude=/host-rootfs \
-	--exclude=/support  \
-	--exclude=/etc/mtab \
-	--exclude=/storage \
-	--exclude=/etc/profile.d/userland_profile.sh \
-	--one-file-system \
-	/ \
-	>> "${MONITOR_FILE_PATH}" || e=$?
+readonly ORDINALY_EXCLUDE_CON="$(\
+	echo "/sys
+		/dev
+		/proc
+		/data
+		/mnt
+		/host-rootfs
+		/support
+		/etc/mtab
+		/storage
+		/etc/profile.d/userland_profile.sh  
+	"\
+	|awk '{ 
+ 		gsub(/^[ \t]+/, "", $0)
+ 		gsub(/[ \t]+$/, "", $0)
+ 		if(!$0) next
+ 		printf "--exclude=%s ", $0
+ 	}'\
+)"
+readonly EXTRA_EXCLUDE_CON=$(\
+	cat "${EXTRA_EXCLUDE_PATH}"\
+	|awk '{ 
+		gsub(/^[ \t]+/, "", $0)
+		gsub(/[ \t]+$/, "", $0)
+		if($0 ~ /^#/) next
+		if(!$0) next
+		print $0
+	}'\
+	| awk '{
+		printf "test -d \x22%s\x22 && echo \x22%s\x22\n", $0, $0
+	}' | bash\
+) 
+awk \
+	-v EXTRA_EXCLUDE_CON="${EXTRA_EXCLUDE_CON}" \
+	-v ORDINALY_EXCLUDE_CON="${ORDINALY_EXCLUDE_CON}" \
+	-v UBUNTU_BACKUP_TEMP_ROOTFS_DIR_PATH="${UBUNTU_BACKUP_TEMP_ROOTFS_DIR_PATH}" \
+	-v ROOTFS_TAR_NAME="${ROOTFS_TAR_NAME}" \
+	-v concurrency=3 \
+	'BEGIN{
+		len_extra_exclude_list = split(EXTRA_EXCLUDE_CON, extra_exclude_list, "\n")
+		cd_cmd="cd /"
+		for(k=1; k <= len_extra_exclude_list; k++){
+			exclude_ops = ""
+	 		for(i=1; i <= len_extra_exclude_list; i++){
+	 			if(k == i) {
+	 				continue
+	 			}
+	 			exclude_ops = sprintf( "%s --exclude=%s ", exclude_ops, extra_exclude_list[i])
+	 		}
+	 		dir_suffix = gensub("/", "___", "g", extra_exclude_list[k])
+	 		make_dir_path = sprintf("%s/%s", UBUNTU_BACKUP_TEMP_ROOTFS_DIR_PATH, dir_suffix)
+	 		gsub(/[/]+/, "/", make_dir_path)
+	 		mkdir_cmd = sprintf("mkdir -p \x22%s\x22", make_dir_path)
+	 		rootfs_path = sprintf("%s/%s", make_dir_path, ROOTFS_TAR_NAME)
+	 		gsub(/[/]+/, "/", rootfs_path)
+	 		tar_cmd_body = sprintf("sudo tar -cpPf \x22%s\x22", rootfs_path)
+	 		total_exclude_con = sprintf("%s %s", ORDINALY_EXCLUDE_CON, exclude_ops)
+	 		tar_ops = "--one-file-system "
+	 		output_cmd = sprintf("echo \x22[%d/%d]\x22", k, len_extra_exclude_list)
+	 		tar_cmd = sprintf("%s %s %s /", \
+	 			tar_cmd_body, \
+	 			total_exclude_con, \
+	 			tar_ops\
+	 		)
+	 		pipe_cmd = "e=$?"
+	 		printf "%s && %s && %s && %s || %s &\n", 
+	 			cd_cmd,
+	 			mkdir_cmd, 
+	 			tar_cmd, 
+	 			output_cmd,
+	 			pipe_cmd
+	 		if(\
+	 			k % concurrency == 0\
+	 			&& k > 0 \
+	 		){
+	 			print "wait"
+	 		}
+		}
+		print "wait"
+	}' | bash >> "${MONITOR_FILE_PATH}" || e=$?
 
-mkdir -p "${UBUNTU_BACKUP_DIR_PATH}"
+rm -rf "${UBUNTU_BACKUP_ROOTFS_DIR_PATH}"
+mkdir "${UBUNTU_BACKUP_ROOTFS_DIR_PATH}"
 echo "cp rootfs.." >> "${MONITOR_FILE_PATH}"
-sudo cp -vf \
-	"${UBUNTU_BACKUP_TEMP_ROOTFS_PATH}" \
+sudo cp -rvf \
+	"${UBUNTU_BACKUP_TEMP_ROOTFS_DIR_PATH}" \
 	"${UBUNTU_BACKUP_DIR_PATH}/" \
 	>> "${MONITOR_FILE_PATH}"
 echo "crean up.." >> "${MONITOR_FILE_PATH}"
-sudo rm \
-	-f "${UBUNTU_BACKUP_TEMP_ROOTFS_PATH}"
+rm -rf "${UBUNTU_BACKUP_TEMP_ROOTFS_DIR_PATH}"
 echo "Comp & Click CANCEL" >> "${MONITOR_FILE_PATH}"
 bash "${NOTI_MANAGER_SHELL_PATH}"

app/src/main/assets/ubuntu_setup/support/ubuntu_manager/extra_excludes.txt

@@ -0,0 +1,17 @@
+/etc
+/home
+/root
+/var
+/usr/bin
+/usr/include
+/usr/lib/aarch64-linux-gnu
+/usr/lib/gcc
+/usr/lib/git-core
+/usr/lib/python3
+/usr/lib/python3.10
+/usr/lib/python3.11
+/usr/lib/systemd
+/usr/libexec
+/usr/local
+/usr/sbin
+/usr/share

app/src/main/assets/ubuntu_setup/support/ubuntu_manager/settingVariables/replaceVariablesTable.tsv

@@ -17,4 +17,5 @@ EXEC_INIT_SHELL_PATH	"${NOTI_SHELL_DIR_PATH}/launch_init.sh"
 // other file path
 UBUNTU_ENV_TSV_PATH	"${SUPPORT_DIR_PATH}/${UBUNTU_ENV_TSV_NAME}"
 MONITOR_DEBUG_FILE_PATH	"${MONITOR_DIR_PATH}/term_1"
-MONITOR_FILE_PATH	"${MONITOR_DIR_PATH}/term_3"
+MONITOR_FILE_PATH	"${MONITOR_DIR_PATH}/term_3"
+EXTRA_EXCLUDE_PATH	"${NOTI_SHELL_DIR_PATH}/extra_excludes.txt"

app/src/main/java/com/puutaro/commandclick/common/variable/extra/UbuntuEnvTsv.kt

@@ -7,14 +7,15 @@ import com.puutaro.commandclick.util.NetworkTool
 
 object UbuntuEnvTsv {
 
-    private val rootfsTarGzName = UbuntuFiles.rootfsTarGzName
     fun makeTsv(
         context: Context?,
     ): String {
         return listOf(
             "WAIT_QUIZ_TSV_NAME" to UbuntuFiles.waitQuizTsvName,
-            "UBUNTU_BACKUP_ROOTFS_PATH" to "${UsePath.cmdclickUbuntuBackupDirPath}/${rootfsTarGzName}",
-            "UBUNTU_BACKUP_TEMP_ROOTFS_PATH" to "${UsePath.cmdclickUbuntuBackupTempDirPath}/${rootfsTarGzName}",
+            "UBUNTU_BACKUP_DIR_PATH" to UsePath.cmdclickUbuntuBackupDirPath,
+            "UBUNTU_BACKUP_ROOTFS_DIR_PATH" to UbuntuFiles.ubuntuBackupRootfsDirPath,
+            "UBUNTU_BACKUP_TEMP_ROOTFS_DIR_PATH" to UbuntuFiles.ubuntuBackupTempRootfsDirPath,
+            "ROOTFS_TAR_NAME" to UbuntuFiles.rootfsTarName,
             "UBUNTU_EXTRA_STARTUP_SHELLS_PATH" to UbuntuFiles.ubuntuExtraStartupShellsTsvPath,
             "MUST_PROCESS_GREP_CMDS_TXT" to UbuntuFiles.mustProcessGrepCmdsTxt,
             "IP_V4_ADDRESS" to NetworkTool.getIpv4Address(context),

app/src/main/java/com/puutaro/commandclick/fragment_lib/terminal_fragment/js_interface/JsUbuntu.kt

@@ -9,26 +9,20 @@ import com.puutaro.commandclick.common.variable.path.UsePath
 import com.puutaro.commandclick.fragment.TerminalFragment
 import com.puutaro.commandclick.fragment_lib.terminal_fragment.js_interface.dialog.JsDialog
 import com.puutaro.commandclick.proccess.broadcast.BroadcastSender
-import com.puutaro.commandclick.proccess.ubuntu.ResAndProcess
 import com.puutaro.commandclick.proccess.ubuntu.Shell2Http
-import com.puutaro.commandclick.proccess.ubuntu.SshManager
 import com.puutaro.commandclick.proccess.ubuntu.UbuntuController
 import com.puutaro.commandclick.proccess.ubuntu.UbuntuFiles
 import com.puutaro.commandclick.proccess.ubuntu.UbuntuProcessChecker
-import com.puutaro.commandclick.util.CcPathTool
-import com.puutaro.commandclick.util.CommandClickVariables
 import com.puutaro.commandclick.util.Intent.CurlManager
 import com.puutaro.commandclick.util.JavaScriptLoadUrl
 import com.puutaro.commandclick.util.file.ReadText
 import com.puutaro.commandclick.util.shell.LinuxCmd
-import io.ktor.http.ContentDisposition.Companion.File
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.delay
 import kotlinx.coroutines.launch
 import kotlinx.coroutines.runBlocking
 import kotlinx.coroutines.withContext
-import java.io.File
 
 
 class JsUbuntu(