User/vchintalapat/reuse prev deployments

images/build/Dockerfiles/gitHubActions.Dockerfile

@@ -78,6 +78,7 @@ RUN apt-get update \
         lzma \
         lzma-dev \
         zlib1g-dev \
+        pigz \
     && rm -rf /var/lib/apt/lists/* \
     # This is the folder containing 'links' to benv and build script generator
     && mkdir -p /opt/oryx

images/runtime/python/template.Dockerfile

@@ -41,6 +41,8 @@ RUN apt-get update \
     && apt-get upgrade -y \
     && apt-get install -y --no-install-recommends \
         xz-utils \
+        lz4 \
+        zstd \
         # Install gcc due to error installing viztracer returning gcc not found
         gcc \
     && rm -rf /var/lib/apt/lists/*

src/BuildScriptGenerator/BaseBashBuildScript.sh.tpl

@@ -187,10 +187,31 @@ then
 	{{ end }}
 
 	{{ if CompressDestinationDir }}
+	echo "Deleting existing content of destination directory '$OLD_DESTINATION_DIR'..."
+	rm -rf "$OLD_DESTINATION_DIR"/output* 2>/dev/null || true
+
 	DESTINATION_DIR="$OLD_DESTINATION_DIR"
 	echo "Compressing content of directory '$preCompressedDestinationDir'..."
 	cd "$preCompressedDestinationDir"
-	tar -zcf "$DESTINATION_DIR/output.tar.gz" .
+
+	COMPRESSION_CMD="${ORYX_COMPRESSION_COMMAND:-gzip}"
+
+	case "$COMPRESSION_CMD" in
+	gzip)
+		tar -zcf "$DESTINATION_DIR/output.tar.gz" .
+		;;
+	lz4)
+		tar -I lz4 -cf "$DESTINATION_DIR/output.tar.lz4" .
+		;;
+	zstd*)
+		tar -I "$COMPRESSION_CMD" -cf "$DESTINATION_DIR/output.tar.zst" .
+		;;
+	*)
+		echo "Unsupported compression command: $COMPRESSION_CMD" 1>&2
+		exit 1
+		;;
+	esac
+
 	echo "Copied the compressed output to '$DESTINATION_DIR'"
 	{{ end }}
 fi

src/BuildScriptGenerator/Options/BuildScriptGeneratorOptions.cs

@@ -98,5 +98,7 @@ public class BuildScriptGeneratorOptions
         public string ImageType { get; set; }
 
         public bool OryxDisablePipUpgrade { get; set; }
+
+        public string OryxCompressionCommand { get; set; }
     }
 }

src/BuildScriptGeneratorCli/Options/BuildScriptGeneratorOptionsSetup.cs

@@ -58,6 +58,7 @@ public void Configure(BuildScriptGeneratorLib.BuildScriptGeneratorOptions option
             options.CustomRequirementsTxtPath = this.GetStringValue(SettingsKeys.CustomRequirementsTxtPath);
             options.CallerId = this.GetStringValue(SettingsKeys.CallerId);
             options.OryxDisablePipUpgrade = this.GetBooleanValue(SettingsKeys.OryxDisablePipUpgrade);
+            options.OryxCompressionCommand = this.GetStringValue(SettingsKeys.OryxCompressionCommand);
 
             // Dynamic install
             options.EnableDynamicInstall = this.GetBooleanValue(SettingsKeys.EnableDynamicInstall);

src/BuildScriptGeneratorCli/SettingsKeys.cs

@@ -75,5 +75,6 @@ public static class SettingsKeys
         public const string DebianFlavor = "DEBIAN_FLAVOR";
         public const string CallerId = "CALLER_ID";
         public const string OryxDisablePipUpgrade = "ORYX_DISABLE_PIP_UPGRADE";
+        public const string OryxCompressionCommand = "ORYX_COMPRESSION_COMMAND";
     }
 }

src/startupscriptgenerator/src/common/compressionHelper.go

@@ -12,7 +12,9 @@ import (
 	"io"
 	"log"
 	"os"
+	"os/exec"
 	"path/filepath"
+	"strings"
 )
 
 // Untar the output file to the provided destination directory
@@ -43,12 +45,41 @@ func ExtractTarball(tarballFile string, destinationDir string) {
 	}
 
 	println(fmt.Sprintf("Extracting '%s' to directory '%s'...", tarballFile, destinationDir))
-	err = untar(destinationDir, tarballFile)
+	err = extractTar(destinationDir, tarballFile)
 	if err != nil {
 		panic(fmt.Sprintf("An error occurred when trying to extract tarball '%s'.", tarballFile))
 	}
 }
 
+func extractTar(dst string, tarballFile string) error {
+    // Detect compression format based on file extension
+    if strings.HasSuffix(tarballFile, ".tar.lz4") {
+        // Use lz4 command-line tool for decompression
+        return untarWithCommand(dst, tarballFile, "lz4")
+    } else if strings.HasSuffix(tarballFile, ".tar.zst") {
+        // Use zstd command-line tool for decompression
+        return untarWithCommand(dst, tarballFile, "zstd")
+    } else if strings.HasSuffix(tarballFile, ".tar.gz") || strings.HasSuffix(tarballFile, ".tgz") {
+        // Use native Go gzip decompression
+        return untar(dst, tarballFile)
+    } else {
+        return fmt.Errorf("unsupported compression format for file: %s", tarballFile)
+    }
+}
+
+// untarWithCommand uses the tar command with specified compression tool
+func untarWithCommand(dst string, tarballFile string, compressionTool string) error {
+    cmd := exec.Command("tar", "-I", compressionTool, "-xf", tarballFile, "-C", dst)
+    cmd.Stdout = os.Stdout
+    cmd.Stderr = os.Stderr
+
+    err := cmd.Run()
+    if err != nil {
+        return fmt.Errorf("failed to extract %s archive: %v", compressionTool, err)
+    }
+    return nil
+}
+
 // Credit to https://medium.com/@skdomino/taring-untaring-files-in-go-6b07cf56bc07
 // Untar takes a destination path and a reader; a tar reader loops over the tarfile
 // creating the file structure at 'dst' along the way, and writing any files

src/startupscriptgenerator/src/python/scriptgenerator.go

@@ -64,9 +64,19 @@ func (gen *PythonStartupScriptGenerator) GenerateEntrypointScript() string {
 
 	if gen.Manifest.CompressDestinationDir == "true" {
 		println("Output is compressed. Extracting it...")
-		tarballFile := filepath.Join(gen.AppPath, "output.tar.gz")
-		common.ExtractTarball(tarballFile, gen.Manifest.SourceDirectoryInBuildContainer)
-		println(fmt.Sprintf("App path is set to '%s'", gen.Manifest.SourceDirectoryInBuildContainer))
+
+		// Try compression formats in order: lz4, zstd, gzip
+		compressionFormats := []string{"output.tar.lz4", "output.tar.zst", "output.tar.gz"}
+
+		for _, format := range compressionFormats {
+			tarballPath := filepath.Join(gen.AppPath, format)
+			if common.PathExists(tarballPath) {
+				println(fmt.Sprintf("Found %s, extracting...", format))
+				common.ExtractTarball(tarballPath, gen.Manifest.SourceDirectoryInBuildContainer)
+				println(fmt.Sprintf("App path is set to '%s'", gen.Manifest.SourceDirectoryInBuildContainer))
+				break
+			}
+		}
 	}
 
 	scriptBuilder.WriteString(fmt.Sprintf("echo 'export APP_PATH=\"%s\"' >> ~/.bashrc\n", gen.getAppPath()))