[Experiment] Evaluate replacing go-yara with yara-x

.github/workflows/go-tests.yaml

@@ -17,7 +17,9 @@ permissions:
 jobs:
   test:
     runs-on: mal-ubuntu-latest-8-core
-
+    env:
+      PKG_CONFIG_PATH: ${{ github.workspace }}
+      LD_LIBRARY_PATH: ${{ github.workspace }}
     steps:
       - uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1
         with:
@@ -35,8 +37,25 @@ jobs:
 
       - name: install dependencies
         run: |
-          sudo apt update && sudo apt install libyara-dev xz-utils -y
+          sudo apt update && sudo apt install xz-utils -y
+
+      - name: Clone yara-x
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          repository: VirusTotal/yara-x
+          path: yara-x
 
+      - name: Setup Rust
+        uses: dtolnay/rust-toolchain@315e265cd78dad1e1dcf3a5074f6d6c47029d5aa # ???
+        with:
+          toolchain: stable
+
+      - name: Install yara-x-capi
+        run: |
+          cd ${{ github.workspace }}/yara-x
+          cargo install cargo-c
+          cargo cinstall -p yara-x-capi --release --pkgconfigdir=${{ github.workspace }} --includedir=${{ github.workspace }} --libdir=${{ github.workspace }}
+          cd ${{ github.workspace }}
       - name: Unit tests
         run: |
           make test

.github/workflows/style.yaml

@@ -130,13 +130,26 @@ jobs:
           go-version-file: go.mod
           check-latest: true
 
-      - name: install libyara-dev
+      - name: Clone yara-x
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          repository: VirusTotal/yara-x
+          path: yara-x
+
+      - name: Setup Rust
+        uses: dtolnay/rust-toolchain@315e265cd78dad1e1dcf3a5074f6d6c47029d5aa # ???
+        with:
+          toolchain: stable
+
+      - name: Install yara-x-capi
         run: |
-          sudo apt update && sudo apt install libyara-dev -y
+          cd ${{ github.workspace }}/yara-x
+          cargo install cargo-c
+          cargo cinstall -p yara-x-capi --release --pkgconfigdir=${{ github.workspace }} --includedir=${{ github.workspace }} --libdir=${{ github.workspace }}
+          cd ${{ github.workspace }}
 
       - name: golangci-lint
         uses: golangci/golangci-lint-action@971e284b6050e8a5849b72094c50ab08da042db8 # v6.1.1
         with:
           version: v1.62.0
           args: --timeout=5m
-

.github/workflows/third-party.yaml

@@ -12,6 +12,9 @@ jobs:
   update:
     if: ${{ github.repository }} == 'chainguard-dev/malcontent'
     runs-on: mal-ubuntu-latest-8-core
+    env:
+      PKG_CONFIG_PATH: ${{ github.workspace }}
+      LD_LIBRARY_PATH: ${{ github.workspace }}
     permissions:
       contents: write
       id-token: write
@@ -28,9 +31,23 @@ jobs:
         with:
           scope: chainguard-dev/malcontent
           identity: third-party
-      - name: Install yara and libyara-dev
+      - name: Clone yara-x
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          repository: VirusTotal/yara-x
+          path: yara-x
+
+      - name: Setup Rust
+        uses: dtolnay/rust-toolchain@315e265cd78dad1e1dcf3a5074f6d6c47029d5aa # ???
+        with:
+          toolchain: stable
+
+      - name: Install yara-x-capi
         run: |
-          sudo apt update && sudo apt install yara libyara-dev -y
+          cd ${{ github.workspace }}/yara-x
+          cargo install cargo-c
+          cargo cinstall -p yara-x-capi --release --pkgconfigdir=${{ github.workspace }} --includedir=${{ github.workspace }} --libdir=${{ github.workspace }}
+          cd ${{ github.workspace }}
       - name: Run make update-third-party
         run: |
           make update-third-party

go.mod

@@ -3,13 +3,13 @@ module github.com/chainguard-dev/malcontent
 go 1.23.3
 
 require (
+	github.com/VirusTotal/yara-x/go v0.10.1-0.20241120200831-c8dbcb8b778b
 	github.com/agext/levenshtein v1.2.3
 	github.com/chainguard-dev/clog v1.5.0
 	github.com/fatih/color v1.18.0
 	github.com/gabriel-vasile/mimetype v1.4.6
 	github.com/google/go-cmp v0.6.0
 	github.com/google/go-containerregistry v0.20.2
-	github.com/hillu/go-yara/v4 v4.3.3
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/shirou/gopsutil/v4 v4.24.10
 	github.com/ulikunitz/xz v0.5.12
@@ -52,4 +52,5 @@ require (
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	golang.org/x/net v0.31.0 // indirect
 	golang.org/x/sys v0.27.0 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
 )

go.sum

@@ -1,3 +1,5 @@
+github.com/VirusTotal/yara-x/go v0.10.1-0.20241120200831-c8dbcb8b778b h1:C7cyOh5jEE+i0u2JHOJMyQ7GVJ5Zfxyf5muRkeAUQFQ=
+github.com/VirusTotal/yara-x/go v0.10.1-0.20241120200831-c8dbcb8b778b/go.mod h1:lgXP/nkYX349MVowrtTtU5hzMdCOWQLv3+wKll9+0F8=
 github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7lmo=
 github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558=
 github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
@@ -32,8 +34,6 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-containerregistry v0.20.2 h1:B1wPJ1SN/S7pB+ZAimcciVD+r+yV/l/DSArMxlbwseo=
 github.com/google/go-containerregistry v0.20.2/go.mod h1:z38EKdKh4h7IP2gSfUUqEvalZBqs6AoLeWfUy34nQC8=
-github.com/hillu/go-yara/v4 v4.3.3 h1:O+7iYTZK20fzsXiJyvA0d529RTdnZCrgS6HdE0O7BMg=
-github.com/hillu/go-yara/v4 v4.3.3/go.mod h1:AHEs/FXVMQKVVlT6iG9d+q1BRr0gq0WoAWZQaZ0gS7s=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
 github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
@@ -71,8 +71,6 @@ github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:Om
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
-github.com/russross/blackfriday v1.6.0 h1:KqfZb0pUVN2lYqZUYRddxF4OR8ZMURnJIG5Y3VRLtww=
-github.com/russross/blackfriday v1.6.0/go.mod h1:ti0ldHuxg49ri4ksnFxlkCfN+hvslNlmVHqNRXXJNAY=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/shirou/gopsutil/v4 v4.24.10 h1:7VOzPtfw/5YDU+jLEoBwXwxJbQetULywoSV4RYY7HkM=
@@ -113,6 +111,8 @@ golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s=
 golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.26.0 h1:WEQa6V3Gja/BhNxg540hBip/kkaYtRg3cxg4oXSw4AU=
 golang.org/x/term v0.26.0/go.mod h1:Si5m1o57C5nBNQo5z1iq+XDijt21BDBDp2bK0QI8e3E=
+google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
+google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=

pkg/action/scan.go

@@ -16,20 +16,20 @@ import (
 	"strings"
 	"sync"
 
+	yarax "github.com/VirusTotal/yara-x/go"
 	"github.com/chainguard-dev/clog"
 	"github.com/chainguard-dev/malcontent/pkg/compile"
 	"github.com/chainguard-dev/malcontent/pkg/malcontent"
 	"github.com/chainguard-dev/malcontent/pkg/programkind"
 	"github.com/chainguard-dev/malcontent/pkg/render"
 	"github.com/chainguard-dev/malcontent/pkg/report"
 
-	"github.com/hillu/go-yara/v4"
 	"golang.org/x/sync/errgroup"
 )
 
 var (
 	// compiledRuleCache are a cache of previously compiled rules.
-	compiledRuleCache *yara.Rules
+	compiledRuleCache *yarax.Rules
 	// compileOnce ensures that we compile rules only once even across threads.
 	compileOnce         sync.Once
 	ErrMatchedCondition = errors.New("matched exit criteria")
@@ -92,7 +92,7 @@ func formatPath(path string) string {
 // scanSinglePath YARA scans a single path and converts it to a fileReport.
 func scanSinglePath(ctx context.Context, c malcontent.Config, path string, ruleFS []fs.FS, absPath string, archiveRoot string) (*malcontent.FileReport, error) {
 	logger := clog.FromContext(ctx)
-	var mrs yara.MatchRules
+	var mrs *yarax.ScanResults
 	logger = logger.With("path", path)
 
 	isArchive := archiveRoot != ""
@@ -111,16 +111,14 @@ func scanSinglePath(ctx context.Context, c malcontent.Config, path string, ruleF
 	}
 	logger = logger.With("mime", mime)
 
-	f, err := os.Open(path)
+	f, err := os.ReadFile(path)
 	if err != nil {
 		return nil, err
 	}
-	defer f.Close()
-	fd := f.Fd()
 
 	// For non-refresh scans, c.Rules will be nil
 	// For refreshes, the rules _should_ be compiled by the time we get here
-	var yrs *yara.Rules
+	var yrs *yarax.Rules
 	if c.Rules == nil {
 		yrs, err = CachedRules(ctx, ruleFS)
 		if err != nil {
@@ -129,12 +127,13 @@ func scanSinglePath(ctx context.Context, c malcontent.Config, path string, ruleF
 	} else {
 		yrs = c.Rules
 	}
-	if err := yrs.ScanFileDescriptor(fd, 0, 0, &mrs); err != nil {
+
+	if mrs, err = yrs.Scan(f); err != nil {
 		logger.Info("skipping", slog.Any("error", err))
 		return &malcontent.FileReport{Path: path, Error: fmt.Sprintf("scan: %v", err)}, nil
 	}
 
-	fr, err := report.Generate(ctx, path, mrs, c, archiveRoot, logger)
+	fr, err := report.Generate(ctx, path, mrs, c, archiveRoot, logger, f)
 	if err != nil {
 		return nil, err
 	}
@@ -240,12 +239,12 @@ func exitIfHitOrMiss(frs *sync.Map, scanPath string, errIfHit bool, errIfMiss bo
 	return nil, nil
 }
 
-func CachedRules(ctx context.Context, fss []fs.FS) (*yara.Rules, error) {
+func CachedRules(ctx context.Context, fss []fs.FS) (*yarax.Rules, error) {
 	if compiledRuleCache != nil {
 		return compiledRuleCache, nil
 	}
 
-	var yrs *yara.Rules
+	var yrs *yarax.Rules
 	var err error
 	compileOnce.Do(func() {
 		yrs, err = compile.Recursive(ctx, fss)

pkg/action/testdata/scan_archive

@@ -79,7 +79,7 @@
                 {
                     "Description": "Contains a table that may be used for XOR decryption",
                     "MatchStrings": [
-                        "56789abcdefghijklmnopqrstuvwxyzABCDE::$ref"
+                        "xor_table::56789abcdefghijklmnopqrstuvwxyzABCDE"
                     ],
                     "RiskScore": 0,
                     "RiskLevel": "NONE",